diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 063d150a..0d6785ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,15 +7,21 @@ test: - ./complete-ci-tests # Test that the parsing of the .txt format didn't change from last -# released version. Ensure that the official tags are included when -# running these tests on forks as well. +# released version. This uses the commit ID of the release tags, +# rather than the release tag itself so that contributor forks do not +# need to include the tags in them for this test to work. +# +# The COMMIT_ID should be bumped after each release, so that the list +# of sed hacks needed does not continuously grow. metadata_v0: image: registry.gitlab.com/fdroid/ci-images-server:latest + variables: + RELEASE_COMMIT_ID: 6d69dcddd95fe08ffe431e305932cfdeafd6fc9d # 1.0.0 script: - - git fetch https://gitlab.com/fdroid/fdroidserver 0.8 + - git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID - cd tests - export GITCOMMIT=`git describe` - - git checkout 0.8 # bump after release + - git checkout $RELEASE_COMMIT_ID - cd .. - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata - cd fdroiddata @@ -25,8 +31,6 @@ metadata_v0: - git checkout $GITCOMMIT - cd fdroiddata - ../tests/dump_internal_metadata_format.py - - sed -i -e '/LiberapayID/d' - metadata/dump_*/*.yaml - diff -uw metadata/dump_* debian_testing: diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 70f9acd8..ef4ad1ee 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -12,6 +12,9 @@ printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \ printf 'APT::Acquire::Retries "20";\n' \ > /etc/apt/apt.conf.d/99acquire-retries +printf 'APT::Periodic::Update-Package-Lists "0";\nAPT::Periodic::Unattended-Upgrade "0";\n' \ + > /etc/apt/apt.conf.d/99no-auto-updates + if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then apt-get -y update apt-get -y install apt-transport-https diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 3b990e92..a431dd52 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -17,13 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import sys import os import shutil import glob import subprocess import re import resource +import sys import tarfile import traceback import time @@ -219,7 +219,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): try: cmd_stdout = chan.makefile('rb', 1024) output = bytes() - output += get_android_tools_version_log(build.ndk_path()).encode() + output += common.get_android_tools_version_log(build.ndk_path()).encode() while not chan.exit_status_ready(): line = cmd_stdout.readline() if line: @@ -290,10 +290,6 @@ def force_gradle_build_tools(build_dir, build_tools): path) -def _get_build_timestamp(): - return time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) - - def transform_first_char(string, method): """Uses method() on the first character of string.""" if len(string) == 0: @@ -433,7 +429,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext log_path = os.path.join(log_dir, common.get_toolsversion_logname(app, build)) with open(log_path, 'w') as f: - f.write(get_android_tools_version_log(build.ndk_path())) + f.write(common.get_android_tools_version_log(build.ndk_path())) else: if build.sudo: logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s' @@ -982,42 +978,6 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, return True -def get_android_tools_versions(ndk_path=None): - '''get a list of the versions of all installed Android SDK/NDK components''' - - global config - sdk_path = config['sdk_path'] - if sdk_path[-1] != '/': - sdk_path += '/' - components = [] - if ndk_path: - ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') - if os.path.isfile(ndk_release_txt): - with open(ndk_release_txt, 'r') as fp: - components.append((os.path.basename(ndk_path), fp.read()[:-1])) - - pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) - for root, dirs, files in os.walk(sdk_path): - if 'source.properties' in files: - source_properties = os.path.join(root, 'source.properties') - with open(source_properties, 'r') as fp: - m = pattern.search(fp.read()) - if m: - components.append((root[len(sdk_path):], m.group(1))) - - return components - - -def get_android_tools_version_log(ndk_path): - '''get a list of the versions of all installed Android SDK/NDK components''' - log = '== Installed Android Tools ==\n\n' - components = get_android_tools_versions(ndk_path) - for name, version in sorted(components): - log += '* ' + name + ' (' + version + ')\n' - - return log - - def parse_commandline(): """Parse the command line. Returns options, parser.""" @@ -1067,12 +1027,13 @@ def parse_commandline(): options = None config = None buildserverid = None -starttime = _get_build_timestamp() +fdroidserverid = None +start_timestamp = time.gmtime() def main(): - global options, config, buildserverid + global options, config, buildserverid, fdroidserverid options, parser = parse_commandline() @@ -1187,10 +1148,10 @@ def main(): for build in app.builds: wikilog = None - build_starttime = _get_build_timestamp() + build_starttime = common.get_wiki_timestamp() tools_version_log = '' if not options.onserver: - tools_version_log = get_android_tools_version_log(build.ndk_path()) + tools_version_log = common.get_android_tools_version_log(build.ndk_path()) try: # For the first build of a particular app, we need to set up @@ -1284,7 +1245,7 @@ def main(): f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' % (build.versionCode, build.versionName, build.commit)) f.write('Build completed at ' - + _get_build_timestamp() + '\n') + + common.get_wiki_timestamp() + '\n') f.write('\n' + tools_version_log + '\n') f.write(str(e)) logging.error("Could not build app %s: %s" % (appid, e)) @@ -1309,9 +1270,9 @@ def main(): newpage = site.Pages[lastbuildpage] with open(os.path.join('tmp', 'fdroidserverid')) as fp: fdroidserverid = fp.read().rstrip() - txt = "* build session started at " + starttime + '\n' \ + txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \ + "* this build started at " + build_starttime + '\n' \ - + "* this build completed at " + _get_build_timestamp() + '\n' \ + + "* this build completed at " + common.get_wiki_timestamp() + '\n' \ + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ + fdroidserverid + ' ' + fdroidserverid + ']\n\n' if buildserverid: @@ -1378,6 +1339,35 @@ def main(): logging.info(ngettext("{} build failed", "{} builds failed", len(failed_apps)).format(len(failed_apps))) + if options.wiki: + wiki_page_path = 'build_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: %s\n" % ' '.join(sys.argv) + txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp) + txt += "* completed at %s\n" % common.get_wiki_timestamp() + if buildserverid: + txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n' + .format(id=buildserverid)) + if fdroidserverid: + txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n' + .format(id=fdroidserverid)) + if os.cpu_count(): + txt += "* host processors: %d\n" % os.cpu_count() + if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK): + with open('/proc/meminfo') as fp: + for line in fp: + m = re.search(r'MemTotal:\s*([0-9].*)', line) + if m: + txt += "* host RAM: %s\n" % m.group(1) + break + txt += "* successful builds: %d\n" % len(build_succeeded) + txt += "* failed builds: %d\n" % len(failed_apps) + txt += "\n\n" + newpage.save(txt, summary='Run log') + newpage = site.Pages['build'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + # hack to ensure this exits, even is some threads are still running sys.stdout.flush() sys.stderr.flush() diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index a303f7fe..876dd2ae 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -23,6 +23,7 @@ import urllib.request import urllib.error import time import subprocess +import sys from argparse import ArgumentParser import traceback import html @@ -510,8 +511,40 @@ def checkupdates_app(app): raise FDroidException("Git commit failed") +def update_wiki(gplaylog, locallog): + if config.get('wiki_server') and config.get('wiki_path'): + try: + import mwclient + site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), + path=config['wiki_path']) + site.login(config['wiki_user'], config['wiki_password']) + + # Write a page with the last build log for this version code + wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += "\n\n" + txt += common.get_android_tools_version_log() + txt += "\n\n" + if gplaylog: + txt += '== --gplay check ==\n\n' + txt += gplaylog + if locallog: + txt += '== local source check ==\n\n' + txt += locallog + newpage.save(txt, summary='Run log') + newpage = site.Pages['checkupdates'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + except Exception as e: + logging.error(_('Error while attempting to publish log: %s') % e) + + config = None options = None +start_timestamp = time.gmtime() def main(): @@ -541,8 +574,10 @@ def main(): apps = common.read_app_args(options.appid, allapps, False) + gplaylog = '' if options.gplay: for appid, app in apps.items(): + gplaylog += '* ' + appid + '\n' version, reason = check_gplay(app) if version is None: if reason == '404': @@ -564,21 +599,28 @@ def main(): else: logging.info("{0} has the same version {1} on the Play Store" .format(common.getappname(app), version)) + update_wiki(gplaylog, None) return + locallog = '' for appid, app in apps.items(): if options.autoonly and app.AutoUpdateMode in ('None', 'Static'): logging.debug(_("Nothing to do for {appid}.").format(appid=appid)) continue - logging.info(_("Processing {appid}").format(appid=appid)) + msg = _("Processing {appid}").format(appid=appid) + logging.info(msg) + locallog += '* ' + msg + '\n' try: checkupdates_app(app) except Exception as e: - logging.error(_("...checkupdate failed for {appid} : {error}") - .format(appid=appid, error=e)) + msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e) + logging.error(msg) + locallog += msg + '\n' + + update_wiki(None, locallog) logging.info(_("Finished")) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 06d40faa..9784ecb6 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3010,3 +3010,47 @@ def get_examples_dir(): examplesdir = prefix + '/examples' return examplesdir + + +def get_wiki_timestamp(timestamp=None): + """Return current time in the standard format for posting to the wiki""" + + if timestamp is None: + timestamp = time.gmtime() + return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) + + +def get_android_tools_versions(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + + global config + sdk_path = config['sdk_path'] + if sdk_path[-1] != '/': + sdk_path += '/' + components = [] + if ndk_path: + ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') + if os.path.isfile(ndk_release_txt): + with open(ndk_release_txt, 'r') as fp: + components.append((os.path.basename(ndk_path), fp.read()[:-1])) + + pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) + for root, dirs, files in os.walk(sdk_path): + if 'source.properties' in files: + source_properties = os.path.join(root, 'source.properties') + with open(source_properties, 'r') as fp: + m = pattern.search(fp.read()) + if m: + components.append((root[len(sdk_path):], m.group(1))) + + return components + + +def get_android_tools_version_log(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + log = '== Installed Android Tools ==\n\n' + components = get_android_tools_versions(ndk_path) + for name, version in sorted(components): + log += '* ' + name + ' (' + version + ')\n' + + return log diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 004a6068..6666830c 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -35,6 +35,7 @@ from .exception import FDroidException config = None options = None +start_timestamp = time.gmtime() BINARY_TRANSPARENCY_DIR = 'binary_transparency' @@ -583,6 +584,28 @@ def push_binary_transparency(git_repo_path, git_remote): origin.push('master') +def update_wiki(): + try: + import mwclient + site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), + path=config['wiki_path']) + site.login(config['wiki_user'], config['wiki_password']) + + # Write a page with the last build log for this version code + wiki_page_path = 'deploy_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += "\n\n" + newpage.save(txt, summary='Run log') + newpage = site.Pages['deploy'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + except Exception as e: + logging.error(_('Error while attempting to publish log: %s') % e) + + def main(): global config, options @@ -723,6 +746,9 @@ def main(): push_binary_transparency(BINARY_TRANSPARENCY_DIR, binary_transparency_remote) + if config.get('wiki_server') and config.get('wiki_path'): + update_wiki() + sys.exit(0) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 60fedfee..9b97fdb0 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -321,7 +321,20 @@ def update_wiki(apps, sortedids, apks): logging.error("...FAILED to create page '{0}': {1}".format(pagename, e)) # Purge server cache to ensure counts are up to date - site.pages['Repository Maintenance'].purge() + site.Pages['Repository Maintenance'].purge() + + # Write a page with the last build log for this version code + wiki_page_path = 'update_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += "\n\n" + txt += common.get_android_tools_version_log() + newpage.save(txt, summary='Run log') + newpage = site.Pages['update'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') def delete_disabled_builds(apps, apkcache, repodirs): @@ -1759,6 +1772,7 @@ def create_metadata_from_template(apk): config = None options = None +start_timestamp = time.gmtime() def main(): diff --git a/jenkins-test b/jenkins-test index 646b0a62..333255af 100755 --- a/jenkins-test +++ b/jenkins-test @@ -56,8 +56,11 @@ test -d archive || mkdir archive # when everything is copied over to run on SIGN machine ../fdroid publish ../fdroid gpgsign -# when everything is copied over to run on BUILD machine +# when everything is copied over to run on BUILD machine, +# which does not have a keyring, only a cached pubkey +echo "repo_pubkey = '308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a'" >> config.py ../fdroid update --nosign +sed -i '/^repo_pubkey = /d' config.py # when everything is copied over to run on SIGN machine ../fdroid signindex --verbose