Merge branch 'log-update-checkupdates-server-to-wiki' into 'master'

Log update/checkupdates/server to wiki

See merge request fdroid/fdroidserver!439

.gitlab-ci.yml

@@ -7,15 +7,21 @@ test:
     - ./complete-ci-tests
 
 # Test that the parsing of the .txt format didn't change from last
-# released version. Ensure that the official tags are included when
+# released version. This uses the commit ID of the release tags,
-# running these tests on forks as well.
+# rather than the release tag itself so that contributor forks do not
+# need to include the tags in them for this test to work.
+#
+# The COMMIT_ID should be bumped after each release, so that the list
+# of sed hacks needed does not continuously grow.
 metadata_v0:
   image: registry.gitlab.com/fdroid/ci-images-server:latest
+  variables:
+    RELEASE_COMMIT_ID: 6d69dcddd95fe08ffe431e305932cfdeafd6fc9d  # 1.0.0
   script:
-    - git fetch https://gitlab.com/fdroid/fdroidserver 0.8
+    - git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID
     - cd tests
     - export GITCOMMIT=`git describe`
-    - git checkout 0.8  # bump after release
+    - git checkout $RELEASE_COMMIT_ID
     - cd ..
     - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
     - cd fdroiddata
@@ -25,8 +31,6 @@ metadata_v0:
     - git checkout $GITCOMMIT
     - cd fdroiddata
     - ../tests/dump_internal_metadata_format.py
-    - sed -i -e '/LiberapayID/d'
-          metadata/dump_*/*.yaml
     - diff -uw metadata/dump_*
 
 debian_testing:

buildserver/provision-apt-get-install

@@ -12,6 +12,9 @@ printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
 printf 'APT::Acquire::Retries "20";\n' \
        > /etc/apt/apt.conf.d/99acquire-retries
 
+printf 'APT::Periodic::Update-Package-Lists "0";\nAPT::Periodic::Unattended-Upgrade "0";\n' \
+       > /etc/apt/apt.conf.d/99no-auto-updates
+
 if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
     apt-get -y update
     apt-get -y install apt-transport-https

fdroidserver/build.py

@@ -17,13 +17,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
 import os
 import shutil
 import glob
 import subprocess
 import re
 import resource
+import sys
 import tarfile
 import traceback
 import time
@@ -219,7 +219,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         try:
             cmd_stdout = chan.makefile('rb', 1024)
             output = bytes()
-            output += get_android_tools_version_log(build.ndk_path()).encode()
+            output += common.get_android_tools_version_log(build.ndk_path()).encode()
             while not chan.exit_status_ready():
                 line = cmd_stdout.readline()
                 if line:
@@ -290,10 +290,6 @@ def force_gradle_build_tools(build_dir, build_tools):
                                path)
 
 
-def _get_build_timestamp():
-    return time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime())
-
-
 def transform_first_char(string, method):
     """Uses method() on the first character of string."""
     if len(string) == 0:
@@ -433,7 +429,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         log_path = os.path.join(log_dir,
                                 common.get_toolsversion_logname(app, build))
         with open(log_path, 'w') as f:
-            f.write(get_android_tools_version_log(build.ndk_path()))
+            f.write(common.get_android_tools_version_log(build.ndk_path()))
     else:
         if build.sudo:
             logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s'
@@ -982,42 +978,6 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
     return True
 
 
-def get_android_tools_versions(ndk_path=None):
-    '''get a list of the versions of all installed Android SDK/NDK components'''
-
-    global config
-    sdk_path = config['sdk_path']
-    if sdk_path[-1] != '/':
-        sdk_path += '/'
-    components = []
-    if ndk_path:
-        ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
-        if os.path.isfile(ndk_release_txt):
-            with open(ndk_release_txt, 'r') as fp:
-                components.append((os.path.basename(ndk_path), fp.read()[:-1]))
-
-    pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
-    for root, dirs, files in os.walk(sdk_path):
-        if 'source.properties' in files:
-            source_properties = os.path.join(root, 'source.properties')
-            with open(source_properties, 'r') as fp:
-                m = pattern.search(fp.read())
-                if m:
-                    components.append((root[len(sdk_path):], m.group(1)))
-
-    return components
-
-
-def get_android_tools_version_log(ndk_path):
-    '''get a list of the versions of all installed Android SDK/NDK components'''
-    log = '== Installed Android Tools ==\n\n'
-    components = get_android_tools_versions(ndk_path)
-    for name, version in sorted(components):
-        log += '* ' + name + ' (' + version + ')\n'
-
-    return log
-
-
 def parse_commandline():
     """Parse the command line. Returns options, parser."""
 
@@ -1067,12 +1027,13 @@ def parse_commandline():
 options = None
 config = None
 buildserverid = None
-starttime = _get_build_timestamp()
+fdroidserverid = None
+start_timestamp = time.gmtime()
 
 
 def main():
 
-    global options, config, buildserverid
+    global options, config, buildserverid, fdroidserverid
 
     options, parser = parse_commandline()
 
@@ -1187,10 +1148,10 @@ def main():
 
         for build in app.builds:
             wikilog = None
-            build_starttime = _get_build_timestamp()
+            build_starttime = common.get_wiki_timestamp()
             tools_version_log = ''
             if not options.onserver:
-                tools_version_log = get_android_tools_version_log(build.ndk_path())
+                tools_version_log = common.get_android_tools_version_log(build.ndk_path())
             try:
 
                 # For the first build of a particular app, we need to set up
@@ -1284,7 +1245,7 @@ def main():
                     f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' %
                             (build.versionCode, build.versionName, build.commit))
                     f.write('Build completed at '
-                            + _get_build_timestamp() + '\n')
+                            + common.get_wiki_timestamp() + '\n')
                     f.write('\n' + tools_version_log + '\n')
                     f.write(str(e))
                 logging.error("Could not build app %s: %s" % (appid, e))
@@ -1309,9 +1270,9 @@ def main():
                     newpage = site.Pages[lastbuildpage]
                     with open(os.path.join('tmp', 'fdroidserverid')) as fp:
                         fdroidserverid = fp.read().rstrip()
-                    txt = "* build session started at " + starttime + '\n' \
+                    txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \
                           + "* this build started at " + build_starttime + '\n' \
-                          + "* this build completed at " + _get_build_timestamp() + '\n' \
+                          + "* this build completed at " + common.get_wiki_timestamp() + '\n' \
                           + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
                           + fdroidserverid + ' ' + fdroidserverid + ']\n\n'
                     if buildserverid:
@@ -1378,6 +1339,35 @@ def main():
         logging.info(ngettext("{} build failed",
                               "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
+    if options.wiki:
+        wiki_page_path = 'build_' + time.strftime('%s', start_timestamp)
+        newpage = site.Pages[wiki_page_path]
+        txt = ''
+        txt += "* command line: <code>%s</code>\n" % ' '.join(sys.argv)
+        txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp)
+        txt += "* completed at %s\n" % common.get_wiki_timestamp()
+        if buildserverid:
+            txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=buildserverid))
+        if fdroidserverid:
+            txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=fdroidserverid))
+        if os.cpu_count():
+            txt += "* host processors: %d\n" % os.cpu_count()
+        if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
+            with open('/proc/meminfo') as fp:
+                for line in fp:
+                    m = re.search(r'MemTotal:\s*([0-9].*)', line)
+                    if m:
+                        txt += "* host RAM: %s\n" % m.group(1)
+                        break
+        txt += "* successful builds: %d\n" % len(build_succeeded)
+        txt += "* failed builds: %d\n" % len(failed_apps)
+        txt += "\n\n"
+        newpage.save(txt, summary='Run log')
+        newpage = site.Pages['build']
+        newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+
     # hack to ensure this exits, even is some threads are still running
     sys.stdout.flush()
     sys.stderr.flush()

fdroidserver/checkupdates.py

@@ -23,6 +23,7 @@ import urllib.request
 import urllib.error
 import time
 import subprocess
+import sys
 from argparse import ArgumentParser
 import traceback
 import html
@@ -510,8 +511,40 @@ def checkupdates_app(app):
                 raise FDroidException("Git commit failed")
 
 
+def update_wiki(gplaylog, locallog):
+    if config.get('wiki_server') and config.get('wiki_path'):
+        try:
+            import mwclient
+            site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+                                 path=config['wiki_path'])
+            site.login(config['wiki_user'], config['wiki_password'])
+
+            # Write a page with the last build log for this version code
+            wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
+            newpage = site.Pages[wiki_page_path]
+            txt = ''
+            txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+            txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+            txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+            txt += "\n\n"
+            txt += common.get_android_tools_version_log()
+            txt += "\n\n"
+            if gplaylog:
+                txt += '== --gplay check ==\n\n'
+                txt += gplaylog
+            if locallog:
+                txt += '== local source check ==\n\n'
+                txt += locallog
+            newpage.save(txt, summary='Run log')
+            newpage = site.Pages['checkupdates']
+            newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+        except Exception as e:
+            logging.error(_('Error while attempting to publish log: %s') % e)
+
+
 config = None
 options = None
+start_timestamp = time.gmtime()
 
 
 def main():
@@ -541,8 +574,10 @@ def main():
 
     apps = common.read_app_args(options.appid, allapps, False)
 
+    gplaylog = ''
     if options.gplay:
         for appid, app in apps.items():
+            gplaylog += '* ' + appid + '\n'
             version, reason = check_gplay(app)
             if version is None:
                 if reason == '404':
@@ -564,21 +599,28 @@ def main():
                     else:
                         logging.info("{0} has the same version {1} on the Play Store"
                                      .format(common.getappname(app), version))
+        update_wiki(gplaylog, None)
         return
 
+    locallog = ''
     for appid, app in apps.items():
 
         if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
             logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
             continue
 
-        logging.info(_("Processing {appid}").format(appid=appid))
+        msg = _("Processing {appid}").format(appid=appid)
+        logging.info(msg)
+        locallog += '* ' + msg + '\n'
 
         try:
             checkupdates_app(app)
         except Exception as e:
-            logging.error(_("...checkupdate failed for {appid} : {error}")
+            msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
-                          .format(appid=appid, error=e))
+            logging.error(msg)
+            locallog += msg + '\n'
+
+    update_wiki(None, locallog)
 
     logging.info(_("Finished"))
 

fdroidserver/common.py

@@ -3010,3 +3010,47 @@ def get_examples_dir():
         examplesdir = prefix + '/examples'
 
     return examplesdir
+
+
+def get_wiki_timestamp(timestamp=None):
+    """Return current time in the standard format for posting to the wiki"""
+
+    if timestamp is None:
+        timestamp = time.gmtime()
+    return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
+
+
+def get_android_tools_versions(ndk_path=None):
+    '''get a list of the versions of all installed Android SDK/NDK components'''
+
+    global config
+    sdk_path = config['sdk_path']
+    if sdk_path[-1] != '/':
+        sdk_path += '/'
+    components = []
+    if ndk_path:
+        ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
+        if os.path.isfile(ndk_release_txt):
+            with open(ndk_release_txt, 'r') as fp:
+                components.append((os.path.basename(ndk_path), fp.read()[:-1]))
+
+    pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
+    for root, dirs, files in os.walk(sdk_path):
+        if 'source.properties' in files:
+            source_properties = os.path.join(root, 'source.properties')
+            with open(source_properties, 'r') as fp:
+                m = pattern.search(fp.read())
+                if m:
+                    components.append((root[len(sdk_path):], m.group(1)))
+
+    return components
+
+
+def get_android_tools_version_log(ndk_path=None):
+    '''get a list of the versions of all installed Android SDK/NDK components'''
+    log = '== Installed Android Tools ==\n\n'
+    components = get_android_tools_versions(ndk_path)
+    for name, version in sorted(components):
+        log += '* ' + name + ' (' + version + ')\n'
+
+    return log

fdroidserver/server.py

@@ -35,6 +35,7 @@ from .exception import FDroidException
 
 config = None
 options = None
+start_timestamp = time.gmtime()
 
 BINARY_TRANSPARENCY_DIR = 'binary_transparency'
 
@@ -583,6 +584,28 @@ def push_binary_transparency(git_repo_path, git_remote):
         origin.push('master')
 
 
+def update_wiki():
+    try:
+        import mwclient
+        site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+                             path=config['wiki_path'])
+        site.login(config['wiki_user'], config['wiki_password'])
+
+        # Write a page with the last build log for this version code
+        wiki_page_path = 'deploy_' + time.strftime('%s', start_timestamp)
+        newpage = site.Pages[wiki_page_path]
+        txt = ''
+        txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+        txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+        txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+        txt += "\n\n"
+        newpage.save(txt, summary='Run log')
+        newpage = site.Pages['deploy']
+        newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+    except Exception as e:
+        logging.error(_('Error while attempting to publish log: %s') % e)
+
+
 def main():
     global config, options
 
@@ -723,6 +746,9 @@ def main():
                 push_binary_transparency(BINARY_TRANSPARENCY_DIR,
                                          binary_transparency_remote)
 
+    if config.get('wiki_server') and config.get('wiki_path'):
+        update_wiki()
+
     sys.exit(0)
 
 

fdroidserver/update.py

@@ -321,7 +321,20 @@ def update_wiki(apps, sortedids, apks):
                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 
     # Purge server cache to ensure counts are up to date
-    site.pages['Repository Maintenance'].purge()
+    site.Pages['Repository Maintenance'].purge()
+
+    # Write a page with the last build log for this version code
+    wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
+    newpage = site.Pages[wiki_page_path]
+    txt = ''
+    txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+    txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+    txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+    txt += "\n\n"
+    txt += common.get_android_tools_version_log()
+    newpage.save(txt, summary='Run log')
+    newpage = site.Pages['update']
+    newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 
 
 def delete_disabled_builds(apps, apkcache, repodirs):
@@ -1759,6 +1772,7 @@ def create_metadata_from_template(apk):
 
 config = None
 options = None
+start_timestamp = time.gmtime()
 
 
 def main():

jenkins-test

@@ -56,8 +56,11 @@ test -d archive || mkdir archive
 # when everything is copied over to run on SIGN machine
 ../fdroid publish
 ../fdroid gpgsign
-# when everything is copied over to run on BUILD machine
+# when everything is copied over to run on BUILD machine,
+# which does not have a keyring, only a cached pubkey
+echo "repo_pubkey = '308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a'" >> config.py
 ../fdroid update --nosign
+sed -i '/^repo_pubkey = /d' config.py
 # when everything is copied over to run on SIGN machine
 ../fdroid signindex --verbose