diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 2ec3d2f9..7123b8a6 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -115,6 +115,7 @@ packages=" scons sqlite3 subversion + sudo swig unzip xsltproc diff --git a/completion/bash-completion b/completion/bash-completion index d679fbd9..f5a8fdb2 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -60,20 +60,14 @@ __apk_vercode() { } __vercode() { - local p v - echo $cur | IFS=':' read p v - - COMPREPLY=( $( compgen -P "${p}:" -W "$( while read line; do - if [[ "$line" == "Build Version:"* ]] - then - line="${line#*,}" - printf "${line%%,*} " - elif [[ "$line" == "Build:"* ]] - then - line="${line#*,}" - printf "${line%%,*} " - fi - done < "metadata/${p}.yml" )" -- $cur ) ) + if [ $prev = ":" ]; then + appid="${COMP_WORDS[COMP_CWORD-2]}" + elif [ $cur = ":" ]; then + appid=$prev + cur="" + fi + versionCodes=`sed -En 's,^ +versionCode: +([0-9]+) *$,\1,p' metadata/${appid}.yml` + COMPREPLY=( $( compgen -W "$versionCodes" -- $cur ) ) } __complete_options() { @@ -91,11 +85,16 @@ __complete_build() { opts="-v -q -l -s -t -f -a -w" lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --no-tarball --force --all --wiki --no-refresh" + case "${prev}" in + :) + __vercode + return 0;; + esac case "${cur}" in -*) __complete_options return 0;; - *:*) + :) __vercode return 0;; *) diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index a75236b9..7ffc6b7e 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -137,9 +137,9 @@ def main(): system_langcode, system_encoding = locale.getdefaultlocale() if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): - logging.warn(_("Encoding is set to '{enc}' fdroid might run " - "into encoding issues. Please set it to 'UTF-8' " - "for best results.".format(enc=system_encoding))) + logging.warning(_("Encoding is set to '{enc}' fdroid might run " + "into encoding issues. Please set it to 'UTF-8' " + "for best results.".format(enc=system_encoding))) try: mod.main() diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 6d8e0241..13317701 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -278,7 +278,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log)) logging.debug('retrieved %s', toolsversion_log) except Exception as e: - logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e)) + logging.warning('could not get %s from builder vm: %s' % (toolsversion_log, e)) # Retrieve the built files... logging.info("Retrieving build output...") diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 70567bb9..5a313bf8 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -131,9 +131,9 @@ def main(): if not package: raise FDroidException(_("Couldn't find package ID")) if not versionName: - logging.warn(_("Couldn't find latest version name")) + logging.warning(_('Could not find latest version name')) if not versionCode: - logging.warn(_("Couldn't find latest version code")) + logging.warning(_('Could not find latest version code')) else: raise FDroidException(_("No gradle project could be found. Specify --subdir?")) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 7eb8bbe2..280a503a 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -130,7 +130,7 @@ def main(): if 'sdk_path' in test_config: common.write_to_config(test_config, 'sdk_path', options.android_home) else: - logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') + logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') @@ -209,8 +209,8 @@ def main(): opensc_so = files[0] else: opensc_so = '/usr/lib/opensc-pkcs11.so' - logging.warn('No OpenSC PKCS#11 module found, ' - + 'install OpenSC then edit "opensc-fdroid.cfg"!') + logging.warning('No OpenSC PKCS#11 module found, ' + + 'install OpenSC then edit "opensc-fdroid.cfg"!') with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f: opensc_fdroid = f.read() opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid, diff --git a/fdroidserver/install.py b/fdroidserver/install.py index c526f1ec..c4e4b7dd 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -109,8 +109,8 @@ def main(): continue if fail == "INSTALL_FAILED_ALREADY_EXISTS": - logging.warn(_("'{apkfilename}' is already installed on {dev}.") - .format(apkfilename=apk, dev=dev)) + logging.warning(_('"{apkfilename}" is already installed on {dev}.') + .format(apkfilename=apk, dev=dev)) else: raise FDroidException(_("Failed to install '{apkfilename}' on {dev}: {error}") .format(apkfilename=apk, dev=dev, error=fail)) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 30084695..6560c4da 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -159,7 +159,7 @@ def scan_source(build_dir, build=metadata.Build()): def warnproblem(what, path_in_build_dir): if toignore(path_in_build_dir): return - logging.warn('Found %s at %s' % (what, path_in_build_dir)) + logging.warning('Found %s at %s' % (what, path_in_build_dir)) if json_per_build is not None: json_per_build['warnings'].append([what, path_in_build_dir]) @@ -356,8 +356,8 @@ def main(): json_per_appid['current-source-state'] = json_per_build count = scan_source(build_dir) if count > 0: - logging.warn(_('Scanner found {count} problems in {appid}:') - .format(count=count, appid=appid)) + logging.warning(_('Scanner found {count} problems in {appid}:') + .format(count=count, appid=appid)) probcount += count app.builds = [] @@ -378,19 +378,19 @@ def main(): count = scan_source(build_dir, build) if count > 0: - logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:') - .format(count=count, appid=appid, versionCode=build.versionCode)) + logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:') + .format(count=count, appid=appid, versionCode=build.versionCode)) probcount += count except BuildException as be: - logging.warn("Could not scan app %s due to BuildException: %s" % ( + logging.warning('Could not scan app %s due to BuildException: %s' % ( appid, be)) probcount += 1 except VCSException as vcse: - logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) + logging.warning('VCS error while scanning app %s: %s' % (appid, vcse)) probcount += 1 except Exception: - logging.warn("Could not scan app %s due to unknown error: %s" % ( + logging.warning('Could not scan app %s due to unknown error: %s' % ( appid, traceback.format_exc())) probcount += 1 diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 313ea424..cc930eb6 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -207,7 +207,7 @@ def update_awsbucket_libcloud(repo_section): s3url = 's3://' + awsbucket + '/' + obj.name logging.info(' deleting ' + s3url) if not driver.delete_object(obj): - logging.warn('Could not delete ' + s3url) + logging.warning('Could not delete ' + s3url) upload = True if upload: @@ -229,7 +229,7 @@ def update_awsbucket_libcloud(repo_section): object_name, obj = objs.popitem() s3url = 's3://' + awsbucket + '/' + object_name if object_name.startswith(upload_dir): - logging.warn(' deleting ' + s3url) + logging.warning(' deleting ' + s3url) driver.delete_object(obj) else: logging.info(' skipping ' + s3url) @@ -773,8 +773,9 @@ def main(): and not config.get('binary_transparency_remote') \ and not config.get('virustotal_apikey') \ and local_copy_dir is None: - logging.warn(_('No option set! Edit your config.py to set at least one of these:') - + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey, androidobservatory, or binary_transparency_remote') + logging.warning(_('No option set! Edit your config.py to set at least one of these:') + + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' + + 'virustotal_apikey, androidobservatory, or binary_transparency_remote') sys.exit(1) repo_sections = ['repo'] diff --git a/fdroidserver/signatures.py b/fdroidserver/signatures.py index 03b2d6f2..79bea6a1 100644 --- a/fdroidserver/signatures.py +++ b/fdroidserver/signatures.py @@ -76,7 +76,9 @@ def extract(options): if tmp_apk and os.path.exists(tmp_apk): os.remove(tmp_apk) else: - logging.warn(_('refuse downloading via insecure HTTP connection (use HTTPS or specify --no-https-check): {apkfilename}').format(apkfilename=apk)) + logging.warning(_('refuse downloading via insecure HTTP connection ' + '(use HTTPS or specify --no-https-check): {apkfilename}') + .format(apkfilename=apk)) except FDroidException as e: logging.warning(_("Failed fetching signatures for '{apkfilename}': {error}") .format(apkfilename=apk, error=e)) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 13947570..4748a7f6 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -366,7 +366,7 @@ def update_wiki(apps, sortedids, apks): else: logging.debug("Page " + page.name + " is unchanged") else: - logging.warn("Deleting page " + page.name) + logging.warning('Deleting page ' + page.name) page.delete('No longer published') for pagename, text in genp.items(): logging.debug("Checking " + pagename) @@ -2162,15 +2162,15 @@ def main(): msg = _("{apkfilename} ({appid}) has no metadata!") \ .format(apkfilename=apk['apkName'], appid=apk['packageName']) if options.delete_unknown: - logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}") - .format(apkfilename=apk['apkName'])) + logging.warning(msg + '\n\t' + _("deleting: repo/{apkfilename}") + .format(apkfilename=apk['apkName'])) rmf = os.path.join(repodirs[0], apk['apkName']) if not os.path.exists(rmf): logging.error(_("Could not find {path} to remove it").format(path=rmf)) else: os.remove(rmf) else: - logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it.")) + logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.')) copy_triple_t_store_metadata(apps) insert_obbs(repodirs[0], apps, apks) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 829d22ca..bd8aae60 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -117,7 +117,7 @@ def get_build_vm(srvdir, provider=None): logging.debug('build vm provider \'virtualbox\' selected') return VirtualboxBuildVm(abssrvdir) else: - logging.warn('build vm provider not supported: \'%s\'', provider) + logging.warning('build vm provider not supported: \'%s\'', provider) # try guessing provider from installed software kvm_installed = shutil.which('kvm') is not None @@ -451,8 +451,8 @@ class LibvirtBuildVm(FDroidBuildVm): os.remove('box.img') else: - logging.warn("could not connect to storage-pool 'default', " - "skip packaging buildserver box") + logging.warning("could not connect to storage-pool 'default', " + "skip packaging buildserver box") def box_add(self, boxname, boxfile, force=True): boximg = '%s_vagrant_box_image_0.img' % (boxname) diff --git a/gradlew-fdroid b/gradlew-fdroid index 6ca320d4..e35da603 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -168,6 +168,7 @@ for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do if [[ $line == 'distributionUrl='* ]]; then wrapper_ver=${line#*/gradle-} wrapper_ver=${wrapper_ver%-*.zip} + wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds break 2 fi done < $f @@ -186,9 +187,11 @@ for f in {.,..}/build.gradle{,.kts}; do if [[ -z "$plugin_pver" && $line == *'com.android.tools.build:gradle:'* ]]; then plugin_pver=${line#*[\'\"]com.android.tools.build:gradle:} plugin_pver=${plugin_pver%[\'\"]*} + plugin_pver=$(printf $plugin_pver | tr -d '\r') # strip Windows linefeeds elif [[ -z "$wrapper_ver" && $line == *'gradleVersion = '* ]]; then wrapper_ver=${line#*gradleVersion*=*[\'\"]} wrapper_ver=${wrapper_ver%[\'\"]*} + wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds fi done < $f done diff --git a/tests/common.TestCase b/tests/common.TestCase index 93fbedd3..4dc51e5c 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1315,6 +1315,11 @@ class CommonTest(unittest.TestCase): self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA')) def test_run_yamllint_wellformed(self): + try: + import yamllint.config + yamllint.config # make pyflakes ignore this + except ImportError: + self.skipTest('yamllint not installed') with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('wellformed.yml', 'w') as f: f.write(textwrap.dedent('''\ @@ -1328,6 +1333,11 @@ class CommonTest(unittest.TestCase): self.assertEqual(result, '') def test_run_yamllint_malformed(self): + try: + import yamllint.config + yamllint.config # make pyflakes ignore this + except ImportError: + self.skipTest('yamllint not installed') with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('malformed.yml', 'w') as f: f.write(textwrap.dedent('''\ diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index c6c1232c..4392664e 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -53,6 +53,33 @@ class ScannerTest(unittest.TestCase): self.assertEqual(should, fatal_problems, "%s should have %d errors!" % (d, should)) + def test_get_gradle_compile_commands(self): + test_files = [ + ('source-files/fdroid/fdroidclient/build.gradle', 'yes', 17), + ('source-files/com.nextcloud.client/build.gradle', 'generic', 24), + ('source-files/com.kunzisoft.testcase/build.gradle', 'libre', 4), + ('source-files/cn.wildfirechat.chat/chat/build.gradle', 'yes', 33), + ('source-files/org.tasks/app/build.gradle.kts', 'generic', 39), + ('source-files/at.bitfire.davdroid/build.gradle', 'standard', 16), + ('source-files/se.manyver/android/app/build.gradle', 'indie', 29), + ('source-files/osmandapp/osmand/build.gradle', 'free', 5), + ('source-files/eu.siacs.conversations/build.gradle', 'free', 23), + ('source-files/org.mozilla.rocket/app/build.gradle', 'focus', 42), + ] + + for f, flavor, count in test_files: + i = 0 + build = fdroidserver.metadata.Build() + build.gradle = [flavor] + regexs = fdroidserver.scanner.get_gradle_compile_commands(build) + with open(f) as fp: + for line in fp.readlines(): + for regex in regexs: + m = regex.match(line) + if m: + i += 1 + self.assertEqual(count, i) + def test_build_local_scanner(self): """`fdroid build` calls scanner functions, test them here""" testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) diff --git a/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties b/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a7a1a8ca --- /dev/null +++ b/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jan 30 10:59:12 CET 2019 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip diff --git a/tests/test-gradlew-fdroid b/tests/test-gradlew-fdroid index d7f9884b..2a34f971 100755 --- a/tests/test-gradlew-fdroid +++ b/tests/test-gradlew-fdroid @@ -19,8 +19,13 @@ exit_value=0 basedir=$(cd $(dirname $0)/..; pwd) export https_proxy=127.7.7.7:7 # fake proxy to block downloading +# force test file to have Windows linefeeds +sed -i -e $'s/\r$//' -e $'s/$/\r/' \ + $basedir/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties + run_test osmandapp/osmand 2.2.1 run_test com.integreight.onesheeld 3.3 run_test se.manyver/android 5.5 +run_test yuriykulikov/AlarmClock 5.1.1 exit $exit_value