kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-02 09:10:11 +02:00
Code Activity

Merge branch 'log-update-checkupdates-server-to-wiki' into 'master'

Browse Source 
Log update/checkupdates/server to wiki

See merge request fdroid/fdroidserver!439
This commit is contained in:
Hans-Christoph Steiner 2018-01-22 13:29:45 +00:00
commit 61bb74a369
8 changed files with 187 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.gitlab-ci.yml
View File

@ -7,15 +7,21 @@ test:
- ./complete-ci-tests
# Test that the parsing of the .txt format didn't change from last
# released version. Ensure that the official tags are included when
# running these tests on forks as well.
# released version. This uses the commit ID of the release tags,
# rather than the release tag itself so that contributor forks do not
# need to include the tags in them for this test to work.
#
# The COMMIT_ID should be bumped after each release, so that the list
# of sed hacks needed does not continuously grow.
metadata_v0:
image: registry.gitlab.com/fdroid/ci-images-server:latest
variables:
RELEASE_COMMIT_ID: 6d69dcddd95fe08ffe431e305932cfdeafd6fc9d # 1.0.0
script:
- git fetch https://gitlab.com/fdroid/fdroidserver 0.8
- git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID
- cd tests
- export GITCOMMIT=`git describe`
- git checkout 0.8 # bump after release
- git checkout $RELEASE_COMMIT_ID
- cd ..
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
- cd fdroiddata
@ -25,8 +31,6 @@ metadata_v0:
- git checkout $GITCOMMIT
- cd fdroiddata
- ../tests/dump_internal_metadata_format.py
- sed -i -e '/LiberapayID/d'
metadata/dump_*/*.yaml
- diff -uw metadata/dump_*
debian_testing:

3
buildserver/provision-apt-get-install
View File

@ -12,6 +12,9 @@ printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
printf 'APT::Acquire::Retries "20";\n' \
> /etc/apt/apt.conf.d/99acquire-retries
printf 'APT::Periodic::Update-Package-Lists "0";\nAPT::Periodic::Unattended-Upgrade "0";\n' \
> /etc/apt/apt.conf.d/99no-auto-updates
if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
apt-get -y update
apt-get -y install apt-transport-https

90
fdroidserver/build.py
View File

@ -17,13 +17,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import os
import shutil
import glob
import subprocess
import re
import resource
import sys
import tarfile
import traceback
import time
@ -219,7 +219,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
try:
cmd_stdout = chan.makefile('rb', 1024)
output = bytes()
output += get_android_tools_version_log(build.ndk_path()).encode()
output += common.get_android_tools_version_log(build.ndk_path()).encode()
while not chan.exit_status_ready():
line = cmd_stdout.readline()
if line:
@ -290,10 +290,6 @@ def force_gradle_build_tools(build_dir, build_tools):
path)
def _get_build_timestamp():
return time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime())
def transform_first_char(string, method):
"""Uses method() on the first character of string."""
if len(string) == 0:
@ -433,7 +429,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
log_path = os.path.join(log_dir,
common.get_toolsversion_logname(app, build))
with open(log_path, 'w') as f:
f.write(get_android_tools_version_log(build.ndk_path()))
f.write(common.get_android_tools_version_log(build.ndk_path()))
else:
if build.sudo:
logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s'
@ -982,42 +978,6 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
return True
def get_android_tools_versions(ndk_path=None):
'''get a list of the versions of all installed Android SDK/NDK components'''
global config
sdk_path = config['sdk_path']
if sdk_path[-1] != '/':
sdk_path += '/'
components = []
if ndk_path:
ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
if os.path.isfile(ndk_release_txt):
with open(ndk_release_txt, 'r') as fp:
components.append((os.path.basename(ndk_path), fp.read()[:-1]))
pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
for root, dirs, files in os.walk(sdk_path):
if 'source.properties' in files:
source_properties = os.path.join(root, 'source.properties')
with open(source_properties, 'r') as fp:
m = pattern.search(fp.read())
if m:
components.append((root[len(sdk_path):], m.group(1)))
return components
def get_android_tools_version_log(ndk_path):
'''get a list of the versions of all installed Android SDK/NDK components'''
log = '== Installed Android Tools ==\n\n'
components = get_android_tools_versions(ndk_path)
for name, version in sorted(components):
log += '* ' + name + ' (' + version + ')\n'
return log
def parse_commandline():
"""Parse the command line. Returns options, parser."""
@ -1067,12 +1027,13 @@ def parse_commandline():
options = None
config = None
buildserverid = None
starttime = _get_build_timestamp()
fdroidserverid = None
start_timestamp = time.gmtime()
def main():
global options, config, buildserverid
global options, config, buildserverid, fdroidserverid
options, parser = parse_commandline()
@ -1187,10 +1148,10 @@ def main():
for build in app.builds:
wikilog = None
build_starttime = _get_build_timestamp()
build_starttime = common.get_wiki_timestamp()
tools_version_log = ''
if not options.onserver:
tools_version_log = get_android_tools_version_log(build.ndk_path())
tools_version_log = common.get_android_tools_version_log(build.ndk_path())
try:
# For the first build of a particular app, we need to set up
@ -1284,7 +1245,7 @@ def main():
f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' %
(build.versionCode, build.versionName, build.commit))
f.write('Build completed at '
+ _get_build_timestamp() + '\n')
+ common.get_wiki_timestamp() + '\n')
f.write('\n' + tools_version_log + '\n')
f.write(str(e))
logging.error("Could not build app %s: %s" % (appid, e))
@ -1309,9 +1270,9 @@ def main():
newpage = site.Pages[lastbuildpage]
with open(os.path.join('tmp', 'fdroidserverid')) as fp:
fdroidserverid = fp.read().rstrip()
txt = "* build session started at " + starttime + '\n' \
txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \
+ "* this build started at " + build_starttime + '\n' \
+ "* this build completed at " + _get_build_timestamp() + '\n' \
+ "* this build completed at " + common.get_wiki_timestamp() + '\n' \
+ '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
+ fdroidserverid + ' ' + fdroidserverid + ']\n\n'
if buildserverid:
@ -1378,6 +1339,35 @@ def main():
logging.info(ngettext("{} build failed",
"{} builds failed", len(failed_apps)).format(len(failed_apps)))
if options.wiki:
wiki_page_path = 'build_' + time.strftime('%s', start_timestamp)
newpage = site.Pages[wiki_page_path]
txt = ''
txt += "* command line: <code>%s</code>\n" % ' '.join(sys.argv)
txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp)
txt += "* completed at %s\n" % common.get_wiki_timestamp()
if buildserverid:
txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
.format(id=buildserverid))
if fdroidserverid:
txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
.format(id=fdroidserverid))
if os.cpu_count():
txt += "* host processors: %d\n" % os.cpu_count()
if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
with open('/proc/meminfo') as fp:
for line in fp:
m = re.search(r'MemTotal:\s*([0-9].*)', line)
if m:
txt += "* host RAM: %s\n" % m.group(1)
break
txt += "* successful builds: %d\n" % len(build_succeeded)
txt += "* failed builds: %d\n" % len(failed_apps)
txt += "\n\n"
newpage.save(txt, summary='Run log')
newpage = site.Pages['build']
newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
# hack to ensure this exits, even is some threads are still running
sys.stdout.flush()
sys.stderr.flush()

48
fdroidserver/checkupdates.py
View File

@ -23,6 +23,7 @@ import urllib.request
import urllib.error
import time
import subprocess
import sys
from argparse import ArgumentParser
import traceback
import html
@ -510,8 +511,40 @@ def checkupdates_app(app):
raise FDroidException("Git commit failed")
def update_wiki(gplaylog, locallog):
if config.get('wiki_server') and config.get('wiki_path'):
try:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Write a page with the last build log for this version code
wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
newpage = site.Pages[wiki_page_path]
txt = ''
txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
txt += "* completed at " + common.get_wiki_timestamp() + '\n'
txt += "\n\n"
txt += common.get_android_tools_version_log()
txt += "\n\n"
if gplaylog:
txt += '== --gplay check ==\n\n'
txt += gplaylog
if locallog:
txt += '== local source check ==\n\n'
txt += locallog
newpage.save(txt, summary='Run log')
newpage = site.Pages['checkupdates']
newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
except Exception as e:
logging.error(_('Error while attempting to publish log: %s') % e)
config = None
options = None
start_timestamp = time.gmtime()
def main():
@ -541,8 +574,10 @@ def main():
apps = common.read_app_args(options.appid, allapps, False)
gplaylog = ''
if options.gplay:
for appid, app in apps.items():
gplaylog += '* ' + appid + '\n'
version, reason = check_gplay(app)
if version is None:
if reason == '404':
@ -564,21 +599,28 @@ def main():
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
update_wiki(gplaylog, None)
return
locallog = ''
for appid, app in apps.items():
if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
continue
logging.info(_("Processing {appid}").format(appid=appid))
msg = _("Processing {appid}").format(appid=appid)
logging.info(msg)
locallog += '* ' + msg + '\n'
try:
checkupdates_app(app)
except Exception as e:
logging.error(_("...checkupdate failed for {appid} : {error}")
.format(appid=appid, error=e))
msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
logging.error(msg)
locallog += msg + '\n'
update_wiki(None, locallog)
logging.info(_("Finished"))

44
fdroidserver/common.py
View File

@ -3010,3 +3010,47 @@ def get_examples_dir():
examplesdir = prefix + '/examples'
return examplesdir
def get_wiki_timestamp(timestamp=None):
"""Return current time in the standard format for posting to the wiki"""
if timestamp is None:
timestamp = time.gmtime()
return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
def get_android_tools_versions(ndk_path=None):
'''get a list of the versions of all installed Android SDK/NDK components'''
global config
sdk_path = config['sdk_path']
if sdk_path[-1] != '/':
sdk_path += '/'
components = []
if ndk_path:
ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
if os.path.isfile(ndk_release_txt):
with open(ndk_release_txt, 'r') as fp:
components.append((os.path.basename(ndk_path), fp.read()[:-1]))
pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
for root, dirs, files in os.walk(sdk_path):
if 'source.properties' in files:
source_properties = os.path.join(root, 'source.properties')
with open(source_properties, 'r') as fp:
m = pattern.search(fp.read())
if m:
components.append((root[len(sdk_path):], m.group(1)))
return components
def get_android_tools_version_log(ndk_path=None):
'''get a list of the versions of all installed Android SDK/NDK components'''
log = '== Installed Android Tools ==\n\n'
components = get_android_tools_versions(ndk_path)
for name, version in sorted(components):
log += '* ' + name + ' (' + version + ')\n'
return log

26
fdroidserver/server.py
View File

@ -35,6 +35,7 @@ from .exception import FDroidException
config = None
options = None
start_timestamp = time.gmtime()
BINARY_TRANSPARENCY_DIR = 'binary_transparency'
@ -583,6 +584,28 @@ def push_binary_transparency(git_repo_path, git_remote):
origin.push('master')
def update_wiki():
try:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Write a page with the last build log for this version code
wiki_page_path = 'deploy_' + time.strftime('%s', start_timestamp)
newpage = site.Pages[wiki_page_path]
txt = ''
txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
txt += "* completed at " + common.get_wiki_timestamp() + '\n'
txt += "\n\n"
newpage.save(txt, summary='Run log')
newpage = site.Pages['deploy']
newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
except Exception as e:
logging.error(_('Error while attempting to publish log: %s') % e)
def main():
global config, options
@ -723,6 +746,9 @@ def main():
push_binary_transparency(BINARY_TRANSPARENCY_DIR,
binary_transparency_remote)
if config.get('wiki_server') and config.get('wiki_path'):
update_wiki()
sys.exit(0)

16
fdroidserver/update.py
View File

@ -321,7 +321,20 @@ def update_wiki(apps, sortedids, apks):
logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
# Purge server cache to ensure counts are up to date
site.pages['Repository Maintenance'].purge()
site.Pages['Repository Maintenance'].purge()
# Write a page with the last build log for this version code
wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
newpage = site.Pages[wiki_page_path]
txt = ''
txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
txt += "* completed at " + common.get_wiki_timestamp() + '\n'
txt += "\n\n"
txt += common.get_android_tools_version_log()
newpage.save(txt, summary='Run log')
newpage = site.Pages['update']
newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
def delete_disabled_builds(apps, apkcache, repodirs):
@ -1759,6 +1772,7 @@ def create_metadata_from_template(apk):
config = None
options = None
start_timestamp = time.gmtime()
def main():

5
jenkins-test
View File

@ -56,8 +56,11 @@ test -d archive || mkdir archive
# when everything is copied over to run on SIGN machine
../fdroid publish
../fdroid gpgsign
# when everything is copied over to run on BUILD machine
# when everything is copied over to run on BUILD machine,
# which does not have a keyring, only a cached pubkey
echo "repo_pubkey = '308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a'" >> config.py
../fdroid update --nosign
sed -i '/^repo_pubkey = /d' config.py
# when everything is copied over to run on SIGN machine
../fdroid signindex --verbose