diff --git a/fdroid b/fdroid index 9a22f571..9b5e120c 100755 --- a/fdroid +++ b/fdroid @@ -22,21 +22,21 @@ import sys import logging commands = { - "build": "Build a package from source", - "init": "Quickly start a new repository", - "publish": "Sign and place packages in the repo", - "update": "Update repo information for new packages", - "verify": "Verify the integrity of downloaded packages", - "checkupdates": "Check for updates to applications", - "import": "Add a new application from its source code", - "install": "Install built packages on devices", - "readmeta": "Read all the metadata files and exit", - "rewritemeta": "Rewrite all the metadata files", - "lint": "Warn about possible metadata errors", - "scanner": "Scan the source code of a package", - "stats": "Update the stats of the repo", - "server": "Interact with the repo HTTP server", - } + "build": "Build a package from source", + "init": "Quickly start a new repository", + "publish": "Sign and place packages in the repo", + "update": "Update repo information for new packages", + "verify": "Verify the integrity of downloaded packages", + "checkupdates": "Check for updates to applications", + "import": "Add a new application from its source code", + "install": "Install built packages on devices", + "readmeta": "Read all the metadata files and exit", + "rewritemeta": "Rewrite all the metadata files", + "lint": "Warn about possible metadata errors", + "scanner": "Scan the source code of a package", + "stats": "Update the stats of the repo", + "server": "Interact with the repo HTTP server", + } def print_help(): diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 4c7a5bd8..e9e033b5 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -432,8 +432,8 @@ def adapt_gradle(build_dir): logging.info("Adapting build.gradle at %s" % path) FDroidPopen(['sed', '-i', - r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"' - + config['build_tools'] + '"@g', path]) + r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"' + + config['build_tools'] + '"@g', path]) def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver): @@ -449,7 +449,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, - build_dir, srclib_dir, extlib_dir, onserver) + build_dir, srclib_dir, + extlib_dir, onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones @@ -492,7 +493,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % - (app['id'], thisbuild['version']), p.stdout) + (app['id'], thisbuild['version']), p.stdout) logging.info("Getting rid of Gradle wrapper binaries...") for root, dirs, files in os.walk(build_dir): @@ -536,9 +537,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d with open(manifest, 'r') as f: manifestcontent = f.read() manifestcontent = manifestcontent.replace('', - '') + '') with open(manifest, 'w') as f: f.write(manifestcontent) @@ -555,7 +558,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % - (app['id'], thisbuild['version']), p.stdout) + (app['id'], thisbuild['version']), p.stdout) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, ['no']): @@ -595,18 +598,20 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d maven_dir = root_dir mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'], - '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', - '-Dandroid.sign.debug=false', '-Dandroid.release=true', - 'package'] + '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', '-Dandroid.release=true', + 'package'] if 'target' in thisbuild: target = thisbuild["target"].split('-')[1] FDroidPopen(['sed', '-i', - 's@[0-9]*@'+target+'@g', - 'pom.xml'], cwd=root_dir) + 's@[0-9]*@'+target+'@g', + 'pom.xml'], + cwd=root_dir) if '@' in thisbuild['maven']: FDroidPopen(['sed', '-i', - 's@[0-9]*@'+target+'@g', - 'pom.xml'], cwd=maven_dir) + 's@[0-9]*@'+target+'@g', + 'pom.xml'], + cwd=maven_dir) if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] @@ -621,7 +626,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException("Expected to find buildozer-compatible spec at {0}" - .format(spec)) + .format(spec)) defaults = {'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18"} @@ -660,7 +665,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d '--package', app['id'], '--version', bconfig.get('app', 'version'), '--orientation', orientation - ] + ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): @@ -720,20 +725,20 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if any(a in line for a in ('.apk', '.ap_'))]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", - stdout_apk, re.S | re.M) + stdout_apk, re.S | re.M) if not m: m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", - stdout_apk, re.S | re.M) + stdout_apk, re.S | re.M) if not m: m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', - stdout_apk, re.S | re.M) + stdout_apk, re.S | re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif thisbuild['type'] == 'kivy': src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format( - bconfig.get('app', 'title'), bconfig.get('app', 'version')) + bconfig.get('app', 'title'), bconfig.get('app', 'version')) elif thisbuild['type'] == 'gradle': basename = app['id'] dd = build_dir @@ -753,7 +758,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, - re.S | re.M).group(1) + re.S | re.M).group(1) src = os.path.join(bindir, src) elif thisbuild['type'] == 'raw': src = os.path.join(root_dir, thisbuild['output']) @@ -769,9 +774,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) - p = SilentPopen([os.path.join(config['sdk_path'], - 'build-tools', config['build_tools'], 'aapt'), - 'dump', 'badging', src]) + p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools', + config['build_tools'], 'aapt'), + 'dump', 'badging', src]) vercode = None version = None @@ -818,10 +823,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" - " APK: '%s' / '%s', " - " Expected: '%s' / '%s'") - % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) - ) + " APK: '%s' / '%s', " + " Expected: '%s' / '%s'") + % (version, str(vercode), thisbuild['version'], + str(thisbuild['vercode'])) + ) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... @@ -835,7 +841,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, - tmp_dir, repo_dir, vcs, test, server, force, onserver): + tmp_dir, repo_dir, vcs, test, server, force, onserver): """ Build a particular version of an application, if it needs building. @@ -996,7 +1002,7 @@ def main(): if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), - path=config['wiki_path']) + path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... @@ -1020,16 +1026,18 @@ def main(): build_dir = os.path.join('build', app['id']) # Set up vcs interface and make sure we have the latest code... - logging.debug("Getting {0} vcs interface for {1}".format( - app['Repo Type'], app['Repo'])) + logging.debug("Getting {0} vcs interface for {1}" + .format(app['Repo Type'], app['Repo'])) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) first = False logging.debug("Checking " + thisbuild['version']) - if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, - srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, - options.server, options.force, options.onserver): + if trybuild(app, thisbuild, build_dir, output_dir, + also_check_dir, srclib_dir, extlib_dir, + tmp_dir, repo_dir, vcs, options.test, + options.server, options.force, + options.onserver): build_succeeded.append(app) wikilog = "Build succeeded" except BuildException as be: diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 51247e48..d55fef3a 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -138,8 +138,8 @@ def check_tags(app, pattern): if not package or package != appid or not version or not vercode: continue - logging.debug("Manifest exists. Found version {0} ({1})".format( - version, vercode)) + logging.debug("Manifest exists. Found version {0} ({1})" + .format(version, vercode)) if int(vercode) > int(hcode): htag = tag hcode = str(int(vercode)) @@ -340,18 +340,18 @@ def main(): if version is not None: stored = app['Current Version'] if not stored: - logging.info("{0} has no Current Version but has version {1} on the Play Store".format( - common.getappname(app), version)) + logging.info("{0} has no Current Version but has version {1} on the Play Store" + .format(common.getappname(app), version)) elif LooseVersion(stored) < LooseVersion(version): - logging.info("{0} has version {1} on the Play Store, which is bigger than {2}".format( - common.getappname(app), version, stored)) + logging.info("{0} has version {1} on the Play Store, which is bigger than {2}" + .format(common.getappname(app), version, stored)) else: if stored != version: - logging.info("{0} has version {1} on the Play Store, which differs from {2}".format( - common.getappname(app), version, stored)) + logging.info("{0} has version {1} on the Play Store, which differs from {2}" + .format(common.getappname(app), version, stored)) else: - logging.info("{0} has the same version {1} on the Play Store".format( - common.getappname(app), version)) + logging.info("{0} has the same version {1} on the Play Store" + .format(common.getappname(app), version)) return for app in apps: @@ -437,7 +437,7 @@ def main(): flavour = None logging.debug("...fetch auto name from " + app_dir + - ((" (flavour: %s)" % flavour) if flavour else "")) + ((" (flavour: %s)" % flavour) if flavour else "")) new_name = common.fetch_real_name(app_dir, flavour) if new_name: logging.debug("...got autoname '" + new_name + "'") @@ -507,8 +507,7 @@ def main(): metadata.write_metadata(metafile, app) if options.commit: logging.info("Commiting update for " + metafile) - gitcmd = ["git", "commit", "-m", - commitmsg] + gitcmd = ["git", "commit", "-m", commitmsg] if 'auto_author' in config: gitcmd.extend(['--author', config['auto_author']]) gitcmd.extend(["--", metafile]) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5cea12b3..1151b75e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -319,7 +319,7 @@ class vcs: # and remote that directory was created from, allowing us to drop it # automatically if either of those things changes. fdpath = os.path.join(self.local, '..', - '.fdroidvcs-' + os.path.basename(self.local)) + '.fdroidvcs-' + os.path.basename(self.local)) cdata = self.repotype() + ' ' + self.remote writeback = True deleterepo = False @@ -463,7 +463,7 @@ class vcs_git(vcs): p = SilentPopen(['echo "'+'\n'.join(alltags)+'" | \ xargs -I@ git log --format=format:"%at @%n" -1 @ | \ sort -n | awk \'{print $2}\''], - cwd=self.local, shell=True) + cwd=self.local, shell=True) return p.stdout.splitlines()[-number:] @@ -700,9 +700,9 @@ class vcs_bzr(vcs): def retrieve_string(app_dir, string, xmlfiles=None): res_dirs = [ - os.path.join(app_dir, 'res'), - os.path.join(app_dir, 'src/main'), - ] + os.path.join(app_dir, 'res'), + os.path.join(app_dir, 'src/main'), + ] if xmlfiles is None: xmlfiles = [] @@ -731,14 +731,15 @@ def retrieve_string(app_dir, string, xmlfiles=None): # Return list of existing files that will be used to find the highest vercode def manifest_paths(app_dir, flavour): - possible_manifests = [os.path.join(app_dir, 'AndroidManifest.xml'), - os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'), - os.path.join(app_dir, 'src', 'AndroidManifest.xml'), - os.path.join(app_dir, 'build.gradle')] + possible_manifests = \ + [os.path.join(app_dir, 'AndroidManifest.xml'), + os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'), + os.path.join(app_dir, 'src', 'AndroidManifest.xml'), + os.path.join(app_dir, 'build.gradle')] if flavour: possible_manifests.append( - os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml')) + os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml')) return [path for path in possible_manifests if os.path.isfile(path)] @@ -920,7 +921,7 @@ class VCSException(Exception): # it, which may be a subdirectory of the actual project. If you want the base # directory of the project, pass 'basepath=True'. def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, - basepath=False, raw=False, prepare=True, preponly=False): + basepath=False, raw=False, prepare=True, preponly=False): number = None subdir = None @@ -990,7 +991,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir) if p.returncode != 0: raise BuildException("Error running prepare command for srclib %s" - % name, p.stdout) + % name, p.stdout) if basepath: libdir = sdir @@ -1042,7 +1043,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % - (app['id'], build['version']), p.stdout) + (app['id'], build['version']), p.stdout) # Apply patches if any if 'patch' in build: @@ -1060,7 +1061,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= logging.info("Collecting source libraries") for lib in build['srclibs']: srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths, - preponly=onserver)) + preponly=onserver)) for name, number, libpath in srclibpaths: place_srclib(root_dir, int(number) if number else None, libpath) @@ -1086,7 +1087,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # from sdk.dir, if necessary if build['oldsdkloc']: sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props, - re.S | re.M).group(1) + re.S | re.M).group(1) props += "sdk-location=%s\n" % sdkloc else: props += "sdk.dir=%s\n" % config['sdk_path'] @@ -1111,14 +1112,16 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if 'target' in build: n = build["target"].split('-')[1] FDroidPopen(['sed', '-i', - 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g', - 'build.gradle'], cwd=root_dir) + 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g', + 'build.gradle'], + cwd=root_dir) if '@' in build['gradle']: gradle_dir = os.path.join(root_dir, build['gradle'].split('@', 1)[1]) gradle_dir = os.path.normpath(gradle_dir) FDroidPopen(['sed', '-i', - 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g', - 'build.gradle'], cwd=gradle_dir) + 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g', + 'build.gradle'], + cwd=gradle_dir) # Remove forced debuggable flags remove_debuggable_flags(root_dir) @@ -1131,14 +1134,16 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= continue if has_extension(path, 'xml'): p = SilentPopen(['sed', '-i', - 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g', - path]) + 's/android:versionName="[^"]*"/android:versionName="' + + build['version'] + '"/g', + path]) if p.returncode != 0: raise BuildException("Failed to amend manifest") elif has_extension(path, 'gradle'): p = SilentPopen(['sed', '-i', - 's/versionName *=* *"[^"]*"/versionName = "' + build['version'] + '"/g', - path]) + 's/versionName *=* *"[^"]*"/versionName = "' + + build['version'] + '"/g', + path]) if p.returncode != 0: raise BuildException("Failed to amend build.gradle") if build['forcevercode']: @@ -1148,14 +1153,16 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= continue if has_extension(path, 'xml'): p = SilentPopen(['sed', '-i', - 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g', - path]) + 's/android:versionCode="[^"]*"/android:versionCode="' + + build['vercode'] + '"/g', + path]) if p.returncode != 0: raise BuildException("Failed to amend manifest") elif has_extension(path, 'gradle'): p = SilentPopen(['sed', '-i', - 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g', - path]) + 's/versionCode *=* *[0-9]*/versionCode = ' + + build['vercode'] + '/g', + path]) if p.returncode != 0: raise BuildException("Failed to amend build.gradle") @@ -1203,7 +1210,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % - (app['id'], build['version']), p.stdout) + (app['id'], build['version']), p.stdout) updatemode = build.get('update', ['auto']) # Generate (or update) the ant build file, build.xml... @@ -1263,21 +1270,21 @@ def scan_source(build_dir, root_dir, thisbuild): # Common known non-free blobs (always lower case): usual_suspects = [ - re.compile(r'flurryagent', re.IGNORECASE), - re.compile(r'paypal.*mpl', re.IGNORECASE), - re.compile(r'libgoogleanalytics', re.IGNORECASE), - re.compile(r'admob.*sdk.*android', re.IGNORECASE), - re.compile(r'googleadview', re.IGNORECASE), - re.compile(r'googleadmobadssdk', re.IGNORECASE), - re.compile(r'google.*play.*services', re.IGNORECASE), - re.compile(r'crittercism', re.IGNORECASE), - re.compile(r'heyzap', re.IGNORECASE), - re.compile(r'jpct.*ae', re.IGNORECASE), - re.compile(r'youtubeandroidplayerapi', re.IGNORECASE), - re.compile(r'bugsense', re.IGNORECASE), - re.compile(r'crashlytics', re.IGNORECASE), - re.compile(r'ouya.*sdk', re.IGNORECASE), - ] + re.compile(r'flurryagent', re.IGNORECASE), + re.compile(r'paypal.*mpl', re.IGNORECASE), + re.compile(r'libgoogleanalytics', re.IGNORECASE), + re.compile(r'admob.*sdk.*android', re.IGNORECASE), + re.compile(r'googleadview', re.IGNORECASE), + re.compile(r'googleadmobadssdk', re.IGNORECASE), + re.compile(r'google.*play.*services', re.IGNORECASE), + re.compile(r'crittercism', re.IGNORECASE), + re.compile(r'heyzap', re.IGNORECASE), + re.compile(r'jpct.*ae', re.IGNORECASE), + re.compile(r'youtubeandroidplayerapi', re.IGNORECASE), + re.compile(r'bugsense', re.IGNORECASE), + re.compile(r'crashlytics', re.IGNORECASE), + re.compile(r'ouya.*sdk', re.IGNORECASE), + ] scanignore = getpaths(build_dir, thisbuild, 'scanignore') scandelete = getpaths(build_dir, thisbuild, 'scandelete') @@ -1460,9 +1467,9 @@ def isApkDebuggable(apkfile, config): :param apkfile: full path to the apk to check""" - p = SilentPopen([os.path.join(config['sdk_path'], - 'build-tools', config['build_tools'], 'aapt'), - 'dump', 'xmltree', apkfile, 'AndroidManifest.xml']) + p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools', + config['build_tools'], 'aapt'), + 'dump', 'xmltree', apkfile, 'AndroidManifest.xml']) if p.returncode != 0: logging.critical("Failed to get apk manifest information") sys.exit(1) @@ -1522,7 +1529,7 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True): result = PopenResult() p = subprocess.Popen(commands, cwd=cwd, shell=shell, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout_queue = Queue.Queue() stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue) @@ -1549,11 +1556,11 @@ def remove_signing_keys(build_dir): comment = re.compile(r'[ ]*//') signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$') line_matches = [ - re.compile(r'^[\t ]*signingConfig [^ ]*$'), - re.compile(r'.*android\.signingConfigs\..*'), - re.compile(r'.*variant\.outputFile = .*'), - re.compile(r'.*\.readLine\(.*'), - ] + re.compile(r'^[\t ]*signingConfig [^ ]*$'), + re.compile(r'.*android\.signingConfigs\..*'), + re.compile(r'.*variant\.outputFile = .*'), + re.compile(r'.*\.readLine\(.*'), + ] for root, dirs, files in os.walk(build_dir): if 'build.gradle' in files: path = os.path.join(root, 'build.gradle') diff --git a/fdroidserver/import.py b/fdroidserver/import.py index e5d03572..39da7923 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -253,7 +253,7 @@ def main(): spec = os.path.join(root_dir, 'buildozer.spec') if os.path.exists(spec): defaults = {'orientation': 'landscape', 'icon': '', - 'permissions': '', 'android.api': "18"} + 'permissions': '', 'android.api': "18"} bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name') diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 235a6a82..ace07bf8 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -72,13 +72,13 @@ def genkey(keystore, repo_keyalias, password, keydname): common.write_password_file("keystorepass", password) common.write_password_file("keypass", password) p = FDroidPopen(['keytool', '-genkey', - '-keystore', keystore, '-alias', repo_keyalias, - '-keyalg', 'RSA', '-keysize', '4096', - '-sigalg', 'SHA256withRSA', - '-validity', '10000', - '-storepass:file', config['keystorepassfile'], - '-keypass:file', config['keypassfile'], - '-dname', keydname]) + '-keystore', keystore, '-alias', repo_keyalias, + '-keyalg', 'RSA', '-keysize', '4096', + '-sigalg', 'SHA256withRSA', + '-validity', '10000', + '-storepass:file', config['keystorepassfile'], + '-keypass:file', config['keypassfile'], + '-dname', keydname]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to generate key", p.stdout) @@ -265,8 +265,7 @@ def main(): if repo_keyalias is not None: logging.info(' Alias for key in store:\t' + repo_keyalias) logging.info('\nTo complete the setup, add your APKs to "' + - os.path.join(fdroiddir, 'repo') + '"' + -''' + os.path.join(fdroiddir, 'repo') + '"' + ''' then run "fdroid update -c; fdroid update". You might also want to edit "config.py" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 8cc70859..1f16426e 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -27,97 +27,96 @@ config = None options = None regex_warnings = { - 'Web Site': [ - (re.compile(r'.*[^sS]://github\.com/.*'), - "github URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://code\.google\.com/.*'), - "code.google.com URLs should always use https:// not http://"), + 'Web Site': [ + (re.compile(r'.*[^sS]://github\.com/.*'), + "github URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://code\.google\.com/.*'), + "code.google.com URLs should always use https:// not http://"), ], - 'Source Code': [ - (re.compile(r'.*[^sS]://github\.com/.*'), - "github URLs should always use https:// (not http://, git://, or git@)"), - (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'), - "/source is missing"), - (re.compile(r'.*[^sS]://code\.google\.com/.*'), - "code.google.com URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://dl\.google\.com/.*'), - "dl.google.com URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://gitorious\.org/.*'), - "gitorious URLs should always use https:// (not http://, git://, or git@)"), + 'Source Code': [ + (re.compile(r'.*[^sS]://github\.com/.*'), + "github URLs should always use https:// (not http://, git://, or git@)"), + (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'), + "/source is missing"), + (re.compile(r'.*[^sS]://code\.google\.com/.*'), + "code.google.com URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://dl\.google\.com/.*'), + "dl.google.com URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://gitorious\.org/.*'), + "gitorious URLs should always use https:// (not http://, git://, or git@)"), ], - 'Repo': [ - (re.compile(r'.*[^sS]://code\.google\.com/.*'), - "code.google.com URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://dl\.google\.com/.*'), - "dl.google.com URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://github\.com/.*'), - "github URLs should always use https:// (not http://, git://, or git@)"), - (re.compile(r'.*[^sS]://gitorious\.org/.*'), - "gitorious URLs should always use https:// (not http://, git://, or git@)"), - (re.compile(r'.*[^sS]://[^.]*\.googlecode\.com/svn/?.*'), - "Google Code SVN URLs should always use https:// (not http:// or svn://)"), - (re.compile(r'.*[^sS]://svn\.apache\.org/repos/?.*'), - "Apache SVN URLs should always use https:// (not http:// or svn://)"), - (re.compile(r'.*[^sS]://svn\.code\.sf\.net/.*'), - "Sourceforge SVN URLs should always use https:// (not http:// or svn://)"), + 'Repo': [ + (re.compile(r'.*[^sS]://code\.google\.com/.*'), + "code.google.com URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://dl\.google\.com/.*'), + "dl.google.com URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://github\.com/.*'), + "github URLs should always use https:// (not http://, git://, or git@)"), + (re.compile(r'.*[^sS]://gitorious\.org/.*'), + "gitorious URLs should always use https:// (not http://, git://, or git@)"), + (re.compile(r'.*[^sS]://[^.]*\.googlecode\.com/svn/?.*'), + "Google Code SVN URLs should always use https:// (not http:// or svn://)"), + (re.compile(r'.*[^sS]://svn\.apache\.org/repos/?.*'), + "Apache SVN URLs should always use https:// (not http:// or svn://)"), + (re.compile(r'.*[^sS]://svn\.code\.sf\.net/.*'), + "Sourceforge SVN URLs should always use https:// (not http:// or svn://)"), ], - 'Issue Tracker': [ - (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'), - "/issues is missing"), - (re.compile(r'.*[^sS]://code\.google\.com/.*'), - "code.google.com URLs should always use https:// not http://"), - (re.compile(r'.*github\.com/[^/]+/[^/]+[/]*$'), - "/issues is missing"), - (re.compile(r'.*[^sS]://github\.com/.*'), - "github URLs should always use https:// not http://"), - (re.compile(r'.*[^sS]://gitorious\.org/.*'), - "gitorious URLs should always use https:// not http://"), + 'Issue Tracker': [ + (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'), + "/issues is missing"), + (re.compile(r'.*[^sS]://code\.google\.com/.*'), + "code.google.com URLs should always use https:// not http://"), + (re.compile(r'.*github\.com/[^/]+/[^/]+[/]*$'), + "/issues is missing"), + (re.compile(r'.*[^sS]://github\.com/.*'), + "github URLs should always use https:// not http://"), + (re.compile(r'.*[^sS]://gitorious\.org/.*'), + "gitorious URLs should always use https:// not http://"), ], - 'Description': [ - (re.compile(r'[ ]*[*#][^ .]'), - "Invalid bulleted list"), - (re.compile(r'^ '), - "Unnecessary leading space"), + 'Description': [ + (re.compile(r'[ ]*[*#][^ .]'), + "Invalid bulleted list"), + (re.compile(r'^ '), + "Unnecessary leading space"), ], - } regex_pedantic = { - 'Web Site': [ - (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), - "Appending .git is not necessary"), - (re.compile(r'.*code\.google\.com/p/[^/]+/[^w]'), - "Possible incorrect path appended to google code project site"), + 'Web Site': [ + (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), + "Appending .git is not necessary"), + (re.compile(r'.*code\.google\.com/p/[^/]+/[^w]'), + "Possible incorrect path appended to google code project site"), ], - 'Source Code': [ - (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), - "Appending .git is not necessary"), - (re.compile(r'.*code\.google\.com/p/[^/]+/source/.*'), - "/source is often enough on its own"), + 'Source Code': [ + (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'), + "Appending .git is not necessary"), + (re.compile(r'.*code\.google\.com/p/[^/]+/source/.*'), + "/source is often enough on its own"), ], - 'Repo': [ - (re.compile(r'^http://.*'), - "if https:// is available, use it instead of http://"), - (re.compile(r'^svn://.*'), - "if https:// is available, use it instead of svn://"), + 'Repo': [ + (re.compile(r'^http://.*'), + "if https:// is available, use it instead of http://"), + (re.compile(r'^svn://.*'), + "if https:// is available, use it instead of svn://"), ], - 'Issue Tracker': [ - (re.compile(r'.*code\.google\.com/p/[^/]+/issues/.*'), - "/issues is often enough on its own"), - (re.compile(r'.*github\.com/[^/]+/[^/]+/issues/.*'), - "/issues is often enough on its own"), + 'Issue Tracker': [ + (re.compile(r'.*code\.google\.com/p/[^/]+/issues/.*'), + "/issues is often enough on its own"), + (re.compile(r'.*github\.com/[^/]+/[^/]+/issues/.*'), + "/issues is often enough on its own"), ], - 'Summary': [ - (re.compile(r'.*\bandroid\b.*', re.IGNORECASE), - "No need to specify that the app is for Android"), - (re.compile(r'.*\b(app|application)\b.*', re.IGNORECASE), - "No need to specify that the app is... an app"), - (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE), - "No need to specify that the app is Free Software"), - (re.compile(r'.*[a-z0-9][.,!?][ $]'), - "Punctuation should be avoided"), + 'Summary': [ + (re.compile(r'.*\bandroid\b.*', re.IGNORECASE), + "No need to specify that the app is for Android"), + (re.compile(r'.*\b(app|application)\b.*', re.IGNORECASE), + "No need to specify that the app is... an app"), + (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE), + "No need to specify that the app is Free Software"), + (re.compile(r'.*[a-z0-9][.,!?][ $]'), + "Punctuation should be avoided"), ], -} + } def main(): @@ -194,7 +193,6 @@ def main(): pwarn("Summary '%s' probably contains redundant info already in app name '%s'" % ( summary, name)) - # Description size limit desc_chars = sum(len(l) for l in app['Description']) if desc_chars > config['char_limits']['Description']: @@ -213,7 +211,6 @@ def main(): if m.match(l): warn("%s at line '%s': %s" % (f, l, r)) - # Regex pedantic checks in all kinds of fields if options.pedantic: for f in regex_pedantic: diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index cd44cbc1..80297c16 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -61,7 +61,7 @@ app_defaults = { 'Repo': '', 'Requires Root': False, 'No Source Since': '' -} + } # This defines the preferred order for the build items - as in the @@ -73,7 +73,7 @@ ordered_flags = [ 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir', 'antcommand', 'novcheck' -] + ] # Designates a metadata field type and checks that it matches @@ -98,15 +98,15 @@ class FieldType(): for v in values: if not self.compiled.match(v): raise MetaDataException("'%s' is not a valid %s in %s. " - % (v, self.name, appid) + - "Regex pattern: %s" % (self.matching)) + % (v, self.name, appid) + + "Regex pattern: %s" % (self.matching)) def _assert_list(self, values, appid): for v in values: if v not in self.matching: raise MetaDataException("'%s' is not a valid %s in %s. " - % (v, self.name, appid) + - "Possible values: %s" % (", ".join(self.matching))) + % (v, self.name, appid) + + "Possible values: %s" % (", ".join(self.matching))) def check(self, value, appid): if type(value) is not str or not value: @@ -124,65 +124,65 @@ class FieldType(): # Generic value types valuetypes = { 'int': FieldType("Integer", - r'^[1-9][0-9]*$', None, - ['FlattrID'], - ['vercode']), + r'^[1-9][0-9]*$', None, + ['FlattrID'], + ['vercode']), 'http': FieldType("HTTP link", - r'^http[s]?://', None, - ["Web Site", "Source Code", "Issue Tracker", "Donate"], []), + r'^http[s]?://', None, + ["Web Site", "Source Code", "Issue Tracker", "Donate"], []), 'bitcoin': FieldType("Bitcoin address", - r'^[a-zA-Z0-9]{27,34}$', None, - ["Bitcoin"], - []), + r'^[a-zA-Z0-9]{27,34}$', None, + ["Bitcoin"], + []), 'litecoin': FieldType("Litecoin address", - r'^L[a-zA-Z0-9]{33}$', None, - ["Litecoin"], - []), + r'^L[a-zA-Z0-9]{33}$', None, + ["Litecoin"], + []), 'dogecoin': FieldType("Dogecoin address", - r'^D[a-zA-Z0-9]{33}$', None, - ["Dogecoin"], - []), + r'^D[a-zA-Z0-9]{33}$', None, + ["Dogecoin"], + []), 'Bool': FieldType("Boolean", - ['Yes', 'No'], None, - ["Requires Root"], - []), + ['Yes', 'No'], None, + ["Requires Root"], + []), 'bool': FieldType("Boolean", - ['yes', 'no'], None, - [], - ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode', - 'novcheck']), + ['yes', 'no'], None, + [], + ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode', + 'novcheck']), 'Repo Type': FieldType("Repo Type", - ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None, - ["Repo Type"], - []), + ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None, + ["Repo Type"], + []), 'archive': FieldType("Archive Policy", - r'^[0-9]+ versions$', None, - ["Archive Policy"], - []), + r'^[0-9]+ versions$', None, + ["Archive Policy"], + []), 'antifeatures': FieldType("Anti-Feature", - ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',', - ["AntiFeatures"], - []), + ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',', + ["AntiFeatures"], + []), 'autoupdatemodes': FieldType("Auto Update Mode", - r"^(Version .+|None)$", None, - ["Auto Update Mode"], - []), + r"^(Version .+|None)$", None, + ["Auto Update Mode"], + []), 'updatecheckmodes': FieldType("Update Check Mode", - r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None, - ["Update Check Mode"], - []) -} + r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None, + ["Update Check Mode"], + []) + } # Check an app's metadata information for integrity errors @@ -447,7 +447,7 @@ def read_metadata(xref=True, package=None, store=True): description_html(app['Description'], linkres) except Exception, e: raise MetaDataException("Problem with description of " + app['id'] + - " - " + str(e)) + " - " + str(e)) return apps @@ -471,7 +471,7 @@ def metafieldtype(name): def flagtype(name): if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', - 'update', 'scanignore', 'scandelete']: + 'update', 'scanignore', 'scandelete']: return 'list' if name in ['init', 'prebuild', 'build']: return 'script' @@ -510,17 +510,17 @@ def parse_metadata(metafile): def add_buildflag(p, thisbuild): bv = p.split('=', 1) if len(bv) != 2: - raise MetaDataException("Invalid build flag at {0} in {1}". - format(buildlines[0], linedesc)) + raise MetaDataException("Invalid build flag at {0} in {1}" + .format(buildlines[0], linedesc)) pk, pv = bv if pk in thisbuild: - raise MetaDataException("Duplicate definition on {0} in version {1} of {2}". - format(pk, thisbuild['version'], linedesc)) + raise MetaDataException("Duplicate definition on {0} in version {1} of {2}" + .format(pk, thisbuild['version'], linedesc)) pk = pk.lstrip() if pk not in ordered_flags: - raise MetaDataException("Unrecognised build flag at {0} in {1}". - format(p, linedesc)) + raise MetaDataException("Unrecognised build flag at {0} in {1}" + .format(p, linedesc)) t = flagtype(pk) if t == 'list': # Port legacy ';' separators @@ -530,8 +530,8 @@ def parse_metadata(metafile): elif t == 'script': thisbuild[pk] = pv else: - raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % ( - t, p, linedesc)) + raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" + % (t, p, linedesc)) def parse_buildline(lines): value = "".join(lines) @@ -606,8 +606,8 @@ def parse_metadata(metafile): if mode == 3: if not any(line.startswith(s) for s in (' ', '\t')): if 'commit' not in curbuild and 'disable' not in curbuild: - raise MetaDataException("No commit specified for {0} in {1}".format( - curbuild['version'], linedesc)) + raise MetaDataException("No commit specified for {0} in {1}" + .format(curbuild['version'], linedesc)) thisinfo['builds'].append(curbuild) add_comments('build:' + curbuild['version']) mode = 0 @@ -662,8 +662,8 @@ def parse_metadata(metafile): curbuild = {} vv = value.split(',') if len(vv) != 2: - raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'. - format(value, linedesc)) + raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}' + .format(value, linedesc)) curbuild['version'] = vv[0] curbuild['vercode'] = vv[1] buildlines = [] diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 863610d4..8fecb6ec 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -128,35 +128,36 @@ def main(): # See if we already have a key for this application, and # if not generate one... p = FDroidPopen(['keytool', '-list', - '-alias', keyalias, '-keystore', config['keystore'], - '-storepass:file', config['keystorepassfile']]) + '-alias', keyalias, '-keystore', config['keystore'], + '-storepass:file', config['keystorepassfile']]) if p.returncode != 0: logging.info("Key does not exist - generating...") p = FDroidPopen(['keytool', '-genkey', - '-keystore', config['keystore'], '-alias', keyalias, - '-keyalg', 'RSA', '-keysize', '2048', - '-validity', '10000', - '-storepass:file', config['keystorepassfile'], - '-keypass:file', config['keypassfile'], - '-dname', config['keydname']]) + '-keystore', config['keystore'], + '-alias', keyalias, + '-keyalg', 'RSA', '-keysize', '2048', + '-validity', '10000', + '-storepass:file', config['keystorepassfile'], + '-keypass:file', config['keypassfile'], + '-dname', config['keydname']]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to generate key") # Sign the application... p = FDroidPopen(['jarsigner', '-keystore', config['keystore'], - '-storepass:file', config['keystorepassfile'], - '-keypass:file', config['keypassfile'], '-sigalg', - 'MD5withRSA', '-digestalg', 'SHA1', - apkfile, keyalias]) + '-storepass:file', config['keystorepassfile'], + '-keypass:file', config['keypassfile'], '-sigalg', + 'MD5withRSA', '-digestalg', 'SHA1', + apkfile, keyalias]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to sign application") # Zipalign it... p = FDroidPopen([os.path.join(config['sdk_path'], 'tools', 'zipalign'), - '-v', '4', apkfile, - os.path.join(output_dir, apkfilename)]) + '-v', '4', apkfile, + os.path.join(output_dir, apkfilename)]) if p.returncode != 0: raise BuildException("Failed to align application") os.remove(apkfile) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 455691ac..5f7d36d7 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -90,13 +90,14 @@ def main(): # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, thisbuild, - build_dir, srclib_dir, extlib_dir, False) + build_dir, srclib_dir, + extlib_dir, False) # Do the scan... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) for problem in buildprobs: - problems.append(problem + - ' in ' + app['id'] + ' ' + thisbuild['version']) + problems.append(problem + ' in ' + app['id'] + + ' ' + thisbuild['version']) except BuildException as be: msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py index fe1205aa..c2f9835c 100644 --- a/fdroidserver/stats.py +++ b/fdroidserver/stats.py @@ -127,8 +127,8 @@ def main(): appscount = Counter() appsvercount = Counter() logexpr = '(?P[.:0-9a-fA-F]+) - - \[(?P