From 03223ab9df524415e0e60ca4522e366dc9d93f0d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Feb 2017 15:50:58 +0100 Subject: [PATCH 1/7] jenkins-build-makebuildserver: test the whole build/publish workflow For full deployments like f-droid.org, the building happens on a separate machine from the signing. This adds a basic test of that kind of setup. --- jenkins-build-makebuildserver | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index f54f4443..12145a2f 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -62,7 +62,22 @@ else fi cd fdroiddata -echo "build_server_always = True" > config.py + +if [ -z $ANDROID_HOME ]; then + if [ -e ~/.android/bashrc ]; then + . ~/.android/bashrc + else + echo "ANDROID_HOME must be set!" + exit + fi +fi + +../fdroid init --verbose +export GNUPGHOME=$WORKSPACE/tests/gnupghome +echo "gpghome = '$GNUPGHOME'" >> config.py +echo "gpgkey = 'CE71F7FB'" >> config.py +echo "build_server_always = True" >> config.py + # if it can't build fdroid, then its really broken ../fdroid build --verbose --stop --latest org.fdroid.fdroid # Gradle, JNI, preassemble @@ -73,3 +88,13 @@ echo "build_server_always = True" > config.py ../fdroid build --verbose --stop org.fdroid.fdroid:96150 # VLC is important, and uses cmake ../fdroid build --verbose --stop org.videolan.vlc:12000604 +# test OTA update ZIP build and publish +../fdroid build --verbose --stop --latest org.fdroid.fdroid.privileged.ota + +# publish process when building and signing are on separate machines +test -d repo || mkdir repo +test -d archive || mkdir archive +../fdroid publish --verbose +../fdroid gpgsign --verbose +../fdroid update --verbose --nosign +../fdroid signindex --verbose From 0cf08de3473dcf2d0ad3f5332da1013d536f5efa Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Feb 2017 16:26:57 +0100 Subject: [PATCH 2/7] build: fix fdroidserverid in build log post on wiki The carriage return in the fdroidserverid file messes up the wiki format. This was forgotten in 69d39bb30101e8fdca1d4369cd2fd3b0d8a6fbc1 --- fdroidserver/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a100548a..736e7ebe 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1206,7 +1206,7 @@ def main(): lastbuildpage = appid + '/lastbuild_' + build.vercode newpage = site.Pages[lastbuildpage] with open(os.path.join('tmp', 'fdroidserverid')) as fp: - fdroidserverid = fp.read() + fdroidserverid = fp.read().rstrip() txt = "* build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n' \ + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ + fdroidserverid + ' ' + fdroidserverid + ']\n\n' \ From 57c3ba413c012cfd8990d10a861f6f14adcc204b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Feb 2017 23:48:40 +0100 Subject: [PATCH 3/7] log Android SDK/NDK component versions on buildserver ad2b9b99c2a7084e1ef4df06d635c7b63bee89e3 put this in the wrong place, it was running it on the buildserver host rather than in the buildserver VM itself, where the builds actually run. refs #148 --- fdroidserver/build.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 736e7ebe..f353bd15 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -389,7 +389,10 @@ def build_server(app, build, vcs, build_dir, output_dir, force): cmdline += ' --verbose' cmdline += " %s:%s" % (app.id, build.vercode) chan.exec_command('bash --login -c "' + cmdline + '"') + output = bytes() + output += b'== Installed Android Tools ==\n\n' + output += get_android_tools_version_log(build.ndk_path()).encode() while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) @@ -952,9 +955,11 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl return True -def get_android_tools_versions(sdk_path, ndk_path=None): +def get_android_tools_versions(ndk_path=None): '''get a list of the versions of all installed Android SDK/NDK components''' + global config + sdk_path = config['sdk_path'] if sdk_path[-1] != '/': sdk_path += '/' components = [] @@ -976,10 +981,10 @@ def get_android_tools_versions(sdk_path, ndk_path=None): return components -def get_android_tools_version_log(sdk_path, ndk_path): +def get_android_tools_version_log(ndk_path): '''get a list of the versions of all installed Android SDK/NDK components''' log = '' - components = get_android_tools_versions(sdk_path, ndk_path) + components = get_android_tools_versions(ndk_path) for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' @@ -1137,7 +1142,7 @@ def main(): for build in app.builds: wikilog = None tools_version_log = '== Installed Android Tools ==\n\n' - tools_version_log += get_android_tools_version_log(config['sdk_path'], build.ndk_path()) + tools_version_log += get_android_tools_version_log(build.ndk_path()) try: # For the first build of a particular app, we need to set up From 837fc99d74f7694a64f014f3d38a8b07d9e8b3bd Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Feb 2017 23:49:42 +0100 Subject: [PATCH 4/7] build: include buildserverid in build log for wiki --- fdroidserver/build.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index f353bd15..ae3b702a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -246,6 +246,8 @@ def release_vm(): def build_server(app, build, vcs, build_dir, output_dir, force): """Do a build on the build server.""" + global buildserverid + try: paramiko except NameError: @@ -258,6 +260,10 @@ def build_server(app, build, vcs, build_dir, output_dir, force): sshinfo = get_clean_vm() try: + if not buildserverid: + buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', + 'cat /home/vagrant/buildserverid'], + cwd='builder').rstrip() # Open SSH connection... logging.info("Connecting to virtual machine...") @@ -1039,11 +1045,12 @@ def parse_commandline(): options = None config = None +buildserverid = None def main(): - global options, config + global options, config, buildserverid options, parser = parse_commandline() @@ -1214,9 +1221,12 @@ def main(): fdroidserverid = fp.read().rstrip() txt = "* build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n' \ + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ - + fdroidserverid + ' ' + fdroidserverid + ']\n\n' \ - + tools_version_log + '\n\n' \ - + '== Build Log ==\n\n' + wikilog + + fdroidserverid + ' ' + fdroidserverid + ']\n\n' + if options.onserver: + txt += '* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ + + buildserverid + ' ' + buildserverid + ']\n\n' + txt += tools_version_log + '\n\n' + txt += '== Build Log ==\n\n' + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] From 3046156a89199fd6b895cee059e0386fac28eb67 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Feb 2017 12:45:04 +0100 Subject: [PATCH 5/7] jenkins-build-makebuildserver: handle git update of fdroiddata better This prevents attempts to merge when there was rebasing, which can fail. --- jenkins-build-makebuildserver | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 12145a2f..d151968b 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -54,8 +54,9 @@ echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then cd fdroiddata + git remote update -p git checkout master - git pull + git reset --hard origin/master cd .. else git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git fdroiddata From 1d91a400a2d2887a02fc43d871873de8b95d9b46 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 10 Feb 2017 15:53:52 +0100 Subject: [PATCH 6/7] buildserver: update to latest tools and m2repository --- makebuildserver | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makebuildserver b/makebuildserver index c986b4d4..9c11b4de 100755 --- a/makebuildserver +++ b/makebuildserver @@ -121,8 +121,8 @@ if config['apt_package_cache']: config['aptcachedir'] = cachedir + '/apt/archives' cachefiles = [ - ('https://dl.google.com/android/repository/tools_r25.2.2-linux.zip', - '10d5185af8fee80a7ee093585bf84962d0a877b92a8fef489e2030e910f6a360'), + ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip', + '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'), ('https://dl.google.com/android/repository/android_m2repository_r43.zip', '8a9aa55c894f9c79e835e3930654b26cc829c35260463b6868296d65782e18d1'), ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip', From 2a39566906379c7c011d9974af22120618b30454 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 7 Feb 2017 11:05:18 +0100 Subject: [PATCH 7/7] jenkins-build-makebuildserver: remove VLC as test, its too flaky Revert "jenkins-build-makebuildserver: include VLC as a test build" This reverts commit 6debb3ebbf0fdbfeb49f80891fd8ba38cea72cd7. --- jenkins-build-makebuildserver | 2 -- 1 file changed, 2 deletions(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index d151968b..aecf2f4e 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -87,8 +87,6 @@ echo "build_server_always = True" >> config.py ../fdroid build --verbose --stop info.guardianproject.checkey:101 # building old versions should still work ../fdroid build --verbose --stop org.fdroid.fdroid:96150 -# VLC is important, and uses cmake -../fdroid build --verbose --stop org.videolan.vlc:12000604 # test OTA update ZIP build and publish ../fdroid build --verbose --stop --latest org.fdroid.fdroid.privileged.ota