From 6030445be041a03b86a8bcb697fb328e2e25cb03 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 25 May 2020 17:36:58 +0200 Subject: [PATCH 1/6] logging.warn() was deprecated in Python 3.3, use logging.warning() sed -i 's,logging\.warn(,logging.warning(,g' fdroid */*.* https://docs.python.org/3.3/library/logging.html#logging.Logger.warning --- fdroidserver/__main__.py | 6 +++--- fdroidserver/build.py | 2 +- fdroidserver/import.py | 4 ++-- fdroidserver/init.py | 6 +++--- fdroidserver/install.py | 4 ++-- fdroidserver/scanner.py | 16 ++++++++-------- fdroidserver/server.py | 9 +++++---- fdroidserver/signatures.py | 4 +++- fdroidserver/update.py | 8 ++++---- fdroidserver/vmtools.py | 6 +++--- 10 files changed, 34 insertions(+), 31 deletions(-) diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index a75236b9..7ffc6b7e 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -137,9 +137,9 @@ def main(): system_langcode, system_encoding = locale.getdefaultlocale() if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): - logging.warn(_("Encoding is set to '{enc}' fdroid might run " - "into encoding issues. Please set it to 'UTF-8' " - "for best results.".format(enc=system_encoding))) + logging.warning(_("Encoding is set to '{enc}' fdroid might run " + "into encoding issues. Please set it to 'UTF-8' " + "for best results.".format(enc=system_encoding))) try: mod.main() diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 6d8e0241..13317701 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -278,7 +278,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log)) logging.debug('retrieved %s', toolsversion_log) except Exception as e: - logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e)) + logging.warning('could not get %s from builder vm: %s' % (toolsversion_log, e)) # Retrieve the built files... logging.info("Retrieving build output...") diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 70567bb9..5a313bf8 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -131,9 +131,9 @@ def main(): if not package: raise FDroidException(_("Couldn't find package ID")) if not versionName: - logging.warn(_("Couldn't find latest version name")) + logging.warning(_('Could not find latest version name')) if not versionCode: - logging.warn(_("Couldn't find latest version code")) + logging.warning(_('Could not find latest version code')) else: raise FDroidException(_("No gradle project could be found. Specify --subdir?")) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 7eb8bbe2..280a503a 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -130,7 +130,7 @@ def main(): if 'sdk_path' in test_config: common.write_to_config(test_config, 'sdk_path', options.android_home) else: - logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') + logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') @@ -209,8 +209,8 @@ def main(): opensc_so = files[0] else: opensc_so = '/usr/lib/opensc-pkcs11.so' - logging.warn('No OpenSC PKCS#11 module found, ' - + 'install OpenSC then edit "opensc-fdroid.cfg"!') + logging.warning('No OpenSC PKCS#11 module found, ' + + 'install OpenSC then edit "opensc-fdroid.cfg"!') with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f: opensc_fdroid = f.read() opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid, diff --git a/fdroidserver/install.py b/fdroidserver/install.py index c526f1ec..c4e4b7dd 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -109,8 +109,8 @@ def main(): continue if fail == "INSTALL_FAILED_ALREADY_EXISTS": - logging.warn(_("'{apkfilename}' is already installed on {dev}.") - .format(apkfilename=apk, dev=dev)) + logging.warning(_('"{apkfilename}" is already installed on {dev}.') + .format(apkfilename=apk, dev=dev)) else: raise FDroidException(_("Failed to install '{apkfilename}' on {dev}: {error}") .format(apkfilename=apk, dev=dev, error=fail)) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 30084695..6560c4da 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -159,7 +159,7 @@ def scan_source(build_dir, build=metadata.Build()): def warnproblem(what, path_in_build_dir): if toignore(path_in_build_dir): return - logging.warn('Found %s at %s' % (what, path_in_build_dir)) + logging.warning('Found %s at %s' % (what, path_in_build_dir)) if json_per_build is not None: json_per_build['warnings'].append([what, path_in_build_dir]) @@ -356,8 +356,8 @@ def main(): json_per_appid['current-source-state'] = json_per_build count = scan_source(build_dir) if count > 0: - logging.warn(_('Scanner found {count} problems in {appid}:') - .format(count=count, appid=appid)) + logging.warning(_('Scanner found {count} problems in {appid}:') + .format(count=count, appid=appid)) probcount += count app.builds = [] @@ -378,19 +378,19 @@ def main(): count = scan_source(build_dir, build) if count > 0: - logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:') - .format(count=count, appid=appid, versionCode=build.versionCode)) + logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:') + .format(count=count, appid=appid, versionCode=build.versionCode)) probcount += count except BuildException as be: - logging.warn("Could not scan app %s due to BuildException: %s" % ( + logging.warning('Could not scan app %s due to BuildException: %s' % ( appid, be)) probcount += 1 except VCSException as vcse: - logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) + logging.warning('VCS error while scanning app %s: %s' % (appid, vcse)) probcount += 1 except Exception: - logging.warn("Could not scan app %s due to unknown error: %s" % ( + logging.warning('Could not scan app %s due to unknown error: %s' % ( appid, traceback.format_exc())) probcount += 1 diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 313ea424..cc930eb6 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -207,7 +207,7 @@ def update_awsbucket_libcloud(repo_section): s3url = 's3://' + awsbucket + '/' + obj.name logging.info(' deleting ' + s3url) if not driver.delete_object(obj): - logging.warn('Could not delete ' + s3url) + logging.warning('Could not delete ' + s3url) upload = True if upload: @@ -229,7 +229,7 @@ def update_awsbucket_libcloud(repo_section): object_name, obj = objs.popitem() s3url = 's3://' + awsbucket + '/' + object_name if object_name.startswith(upload_dir): - logging.warn(' deleting ' + s3url) + logging.warning(' deleting ' + s3url) driver.delete_object(obj) else: logging.info(' skipping ' + s3url) @@ -773,8 +773,9 @@ def main(): and not config.get('binary_transparency_remote') \ and not config.get('virustotal_apikey') \ and local_copy_dir is None: - logging.warn(_('No option set! Edit your config.py to set at least one of these:') - + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey, androidobservatory, or binary_transparency_remote') + logging.warning(_('No option set! Edit your config.py to set at least one of these:') + + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' + + 'virustotal_apikey, androidobservatory, or binary_transparency_remote') sys.exit(1) repo_sections = ['repo'] diff --git a/fdroidserver/signatures.py b/fdroidserver/signatures.py index 03b2d6f2..79bea6a1 100644 --- a/fdroidserver/signatures.py +++ b/fdroidserver/signatures.py @@ -76,7 +76,9 @@ def extract(options): if tmp_apk and os.path.exists(tmp_apk): os.remove(tmp_apk) else: - logging.warn(_('refuse downloading via insecure HTTP connection (use HTTPS or specify --no-https-check): {apkfilename}').format(apkfilename=apk)) + logging.warning(_('refuse downloading via insecure HTTP connection ' + '(use HTTPS or specify --no-https-check): {apkfilename}') + .format(apkfilename=apk)) except FDroidException as e: logging.warning(_("Failed fetching signatures for '{apkfilename}': {error}") .format(apkfilename=apk, error=e)) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 13947570..4748a7f6 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -366,7 +366,7 @@ def update_wiki(apps, sortedids, apks): else: logging.debug("Page " + page.name + " is unchanged") else: - logging.warn("Deleting page " + page.name) + logging.warning('Deleting page ' + page.name) page.delete('No longer published') for pagename, text in genp.items(): logging.debug("Checking " + pagename) @@ -2162,15 +2162,15 @@ def main(): msg = _("{apkfilename} ({appid}) has no metadata!") \ .format(apkfilename=apk['apkName'], appid=apk['packageName']) if options.delete_unknown: - logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}") - .format(apkfilename=apk['apkName'])) + logging.warning(msg + '\n\t' + _("deleting: repo/{apkfilename}") + .format(apkfilename=apk['apkName'])) rmf = os.path.join(repodirs[0], apk['apkName']) if not os.path.exists(rmf): logging.error(_("Could not find {path} to remove it").format(path=rmf)) else: os.remove(rmf) else: - logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it.")) + logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.')) copy_triple_t_store_metadata(apps) insert_obbs(repodirs[0], apps, apks) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 829d22ca..bd8aae60 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -117,7 +117,7 @@ def get_build_vm(srvdir, provider=None): logging.debug('build vm provider \'virtualbox\' selected') return VirtualboxBuildVm(abssrvdir) else: - logging.warn('build vm provider not supported: \'%s\'', provider) + logging.warning('build vm provider not supported: \'%s\'', provider) # try guessing provider from installed software kvm_installed = shutil.which('kvm') is not None @@ -451,8 +451,8 @@ class LibvirtBuildVm(FDroidBuildVm): os.remove('box.img') else: - logging.warn("could not connect to storage-pool 'default', " - "skip packaging buildserver box") + logging.warning("could not connect to storage-pool 'default', " + "skip packaging buildserver box") def box_add(self, boxname, boxfile, force=True): boximg = '%s_vagrant_box_image_0.img' % (boxname) From 95c3ab2454ffea2afdc944b9e82e184922aebdc5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 26 May 2020 11:13:54 +0200 Subject: [PATCH 2/6] skip yamllint test if yamllint is not installed !721 --- tests/common.TestCase | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/common.TestCase b/tests/common.TestCase index 93fbedd3..4dc51e5c 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1315,6 +1315,11 @@ class CommonTest(unittest.TestCase): self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA')) def test_run_yamllint_wellformed(self): + try: + import yamllint.config + yamllint.config # make pyflakes ignore this + except ImportError: + self.skipTest('yamllint not installed') with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('wellformed.yml', 'w') as f: f.write(textwrap.dedent('''\ @@ -1328,6 +1333,11 @@ class CommonTest(unittest.TestCase): self.assertEqual(result, '') def test_run_yamllint_malformed(self): + try: + import yamllint.config + yamllint.config # make pyflakes ignore this + except ImportError: + self.skipTest('yamllint not installed') with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('malformed.yml', 'w') as f: f.write(textwrap.dedent('''\ From cc5aed3c1fb4b42276059e0b1d258ec8cd8fce4d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 25 May 2020 21:44:19 +0200 Subject: [PATCH 3/6] fix version codes in bash completion for `fdroid build` I'm not sure the old code ever worked, but this works now, and supports .yml. closes #719 --- completion/bash-completion | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index d679fbd9..f5a8fdb2 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -60,20 +60,14 @@ __apk_vercode() { } __vercode() { - local p v - echo $cur | IFS=':' read p v - - COMPREPLY=( $( compgen -P "${p}:" -W "$( while read line; do - if [[ "$line" == "Build Version:"* ]] - then - line="${line#*,}" - printf "${line%%,*} " - elif [[ "$line" == "Build:"* ]] - then - line="${line#*,}" - printf "${line%%,*} " - fi - done < "metadata/${p}.yml" )" -- $cur ) ) + if [ $prev = ":" ]; then + appid="${COMP_WORDS[COMP_CWORD-2]}" + elif [ $cur = ":" ]; then + appid=$prev + cur="" + fi + versionCodes=`sed -En 's,^ +versionCode: +([0-9]+) *$,\1,p' metadata/${appid}.yml` + COMPREPLY=( $( compgen -W "$versionCodes" -- $cur ) ) } __complete_options() { @@ -91,11 +85,16 @@ __complete_build() { opts="-v -q -l -s -t -f -a -w" lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --no-tarball --force --all --wiki --no-refresh" + case "${prev}" in + :) + __vercode + return 0;; + esac case "${cur}" in -*) __complete_options return 0;; - *:*) + :) __vercode return 0;; *) From 1ed4ed61c76ed1482df7559bc2c436011a65e6e3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 27 May 2020 17:40:23 +0200 Subject: [PATCH 4/6] buildserver: explicitly include sudo as a dependency The basebox currently provides sudo, but that may not always be the case. This makes the sudo dependency explicit, so that this provisioning script can also be used in other settings, like GitLab CI. --- buildserver/provision-apt-get-install | 1 + 1 file changed, 1 insertion(+) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 2ec3d2f9..7123b8a6 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -115,6 +115,7 @@ packages=" scons sqlite3 subversion + sudo swig unzip xsltproc From fc885c9b5ce962b09a4a0ef804c8f884d3c95cbc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 27 May 2020 22:03:23 +0200 Subject: [PATCH 5/6] scanner: test for get_gradle_compile_commands() --- tests/scanner.TestCase | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index c6c1232c..4392664e 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -53,6 +53,33 @@ class ScannerTest(unittest.TestCase): self.assertEqual(should, fatal_problems, "%s should have %d errors!" % (d, should)) + def test_get_gradle_compile_commands(self): + test_files = [ + ('source-files/fdroid/fdroidclient/build.gradle', 'yes', 17), + ('source-files/com.nextcloud.client/build.gradle', 'generic', 24), + ('source-files/com.kunzisoft.testcase/build.gradle', 'libre', 4), + ('source-files/cn.wildfirechat.chat/chat/build.gradle', 'yes', 33), + ('source-files/org.tasks/app/build.gradle.kts', 'generic', 39), + ('source-files/at.bitfire.davdroid/build.gradle', 'standard', 16), + ('source-files/se.manyver/android/app/build.gradle', 'indie', 29), + ('source-files/osmandapp/osmand/build.gradle', 'free', 5), + ('source-files/eu.siacs.conversations/build.gradle', 'free', 23), + ('source-files/org.mozilla.rocket/app/build.gradle', 'focus', 42), + ] + + for f, flavor, count in test_files: + i = 0 + build = fdroidserver.metadata.Build() + build.gradle = [flavor] + regexs = fdroidserver.scanner.get_gradle_compile_commands(build) + with open(f) as fp: + for line in fp.readlines(): + for regex in regexs: + m = regex.match(line) + if m: + i += 1 + self.assertEqual(count, i) + def test_build_local_scanner(self): """`fdroid build` calls scanner functions, test them here""" testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) From 7a84679b0d7afaffd244ff71191443a9b56446c8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 29 May 2020 21:55:10 +0200 Subject: [PATCH 6/6] gradlew-fdroid: fix parsing when files have Windows linefeeds The new test file should have Windows linefeeds, if I got the git config correct. --- gradlew-fdroid | 3 +++ .../AlarmClock/gradle/wrapper/gradle-wrapper.properties | 6 ++++++ tests/test-gradlew-fdroid | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties diff --git a/gradlew-fdroid b/gradlew-fdroid index 7664b1aa..97cd5394 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -168,6 +168,7 @@ for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do if [[ $line == 'distributionUrl='* ]]; then wrapper_ver=${line#*/gradle-} wrapper_ver=${wrapper_ver%-*.zip} + wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds break 2 fi done < $f @@ -186,9 +187,11 @@ for f in {.,..}/build.gradle{,.kts}; do if [[ -z "$plugin_pver" && $line == *'com.android.tools.build:gradle:'* ]]; then plugin_pver=${line#*[\'\"]com.android.tools.build:gradle:} plugin_pver=${plugin_pver%[\'\"]*} + plugin_pver=$(printf $plugin_pver | tr -d '\r') # strip Windows linefeeds elif [[ -z "$wrapper_ver" && $line == *'gradleVersion = '* ]]; then wrapper_ver=${line#*gradleVersion*=*[\'\"]} wrapper_ver=${wrapper_ver%[\'\"]*} + wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds fi done < $f done diff --git a/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties b/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a7a1a8ca --- /dev/null +++ b/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jan 30 10:59:12 CET 2019 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip diff --git a/tests/test-gradlew-fdroid b/tests/test-gradlew-fdroid index d7f9884b..2a34f971 100755 --- a/tests/test-gradlew-fdroid +++ b/tests/test-gradlew-fdroid @@ -19,8 +19,13 @@ exit_value=0 basedir=$(cd $(dirname $0)/..; pwd) export https_proxy=127.7.7.7:7 # fake proxy to block downloading +# force test file to have Windows linefeeds +sed -i -e $'s/\r$//' -e $'s/$/\r/' \ + $basedir/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties + run_test osmandapp/osmand 2.2.1 run_test com.integreight.onesheeld 3.3 run_test se.manyver/android 5.5 +run_test yuriykulikov/AlarmClock 5.1.1 exit $exit_value