From 9d44fa79190811d70166bc6813e9d12d7e8e9555 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 14 May 2021 00:03:16 +0200 Subject: [PATCH] gitlab-ci: auto-generate merge request when NDK release found Following the pattern of the gradle bot, this will check the transparency log for any new NDK release. If there are any, it will make a merge request from @fdroid-bot. --- .gitlab-ci.yml | 6 +- tests/ndk-release-checksums.py | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) create mode 100755 tests/ndk-release-checksums.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fae5c80..149b34f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -261,7 +261,7 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" -gradle: +gradle/ndk: image: debian:bullseye <<: *apt-template variables: @@ -276,16 +276,18 @@ gradle: python3-git python3-gitlab python3-requests - # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed + # if this is a merge request fork, then only check if relevant files changed - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroidserver.git; for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do test "$f" == "makebuildserver" && export CHANGED="yes"; test "$f" == "gradlew-fdroid" && export CHANGED="yes"; + test "$f" == "fdroidserver/common.py" && export CHANGED="yes"; done; test -z "$CHANGED" && exit; fi - ./tests/gradle-release-checksums.py + - ./tests/ndk-release-checksums.py # Run an actual build in a simple, faked version of the buildserver guest VM. diff --git a/tests/ndk-release-checksums.py b/tests/ndk-release-checksums.py new file mode 100755 index 00000000..6e99d6a0 --- /dev/null +++ b/tests/ndk-release-checksums.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import git +import gitlab +import json +import os +import re +import requests +import subprocess +from colorama import Fore, Style + + +checksums = None +versions = dict() + +while not checksums: + r = requests.get( + 'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json' + ) + if r.status_code == 200: + checksums = r.json() + +with open('fdroidserver/common.py') as fp: + common_py = fp.read() + +to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group() +if not to_compile: + exit(1) +code = compile(to_compile, '', 'exec') +config = {} +exec(code, None, config) # nosec this is just a CI script + +ndks = [] +errors = 0 +release = None +revision = None +for k, entries in checksums.items(): + if k.startswith('https://dl.google.com/android/repository/android-ndk'): + m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k) + if m: + for entry in entries: + if 'source.properties' in entry: + n = re.search( + r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties'] + ) + if n: + release = m.group(1) + revision = n.group() + ndks.append( + { + 'url': k, + 'release': release, + 'revision': revision, + 'sha256': checksums[k][0]['sha256'], + } + ) + for d in config['NDKS']: + if k == d['url']: + sha256 = d['sha256'] + found = False + for entry in entries: + if sha256 == entry['sha256']: + found = True + if not found: + print( + Fore.RED + + ( + 'ERROR: checksum mismatch: %s != %s' + % (sha256, entry['sha256']) + ) + + Style.RESET_ALL + ) + errors += 1 + +with open('fdroidserver/common.py', 'w') as fp: + fp.write( + common_py.replace( + to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True) + ) + ) + +if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid': + p = subprocess.run(['git', '--no-pager', 'diff']) + print(p.stdout) + exit(errors) + + +# This only runs after commits are pushed to fdroid/fdroidserver +git_repo = git.repo.Repo('.') +modified = git_repo.git().ls_files(modified=True).split() +if git_repo.is_dirty() and 'fdroidserver/common.py' in modified: + branch = git_repo.create_head(os.path.basename(__file__), force=True) + branch.checkout() + git_repo.index.add(['fdroidserver/common.py']) + author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org') + git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author) + project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME') + url = 'https://gitlab-ci-token:%s@%s/%s.git' % ( + os.getenv('PERSONAL_ACCESS_TOKEN'), + os.getenv('CI_SERVER_HOST'), + project_path, + ) + remote_name = 'fdroid-bot' + try: + remote = git_repo.create_remote(remote_name, url) + except git.exc.GitCommandError: + remote = git.remote.Remote(git_repo, remote_name) + remote.set_url(url) + remote.push(force=True) + git.remote.Remote.rm(git_repo, remote_name) + + private_token = os.getenv('PERSONAL_ACCESS_TOKEN') + if not private_token: + print( + Fore.RED + + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!' + + Style.RESET_ALL + ) + exit(1) + gl = gitlab.Gitlab( + os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token + ) + project = gl.projects.get(project_path, lazy=True) + description = ( + 'see ' + '\n\n

generated by GitLab CI Job #%s

' + % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID')) + ) + mr = project.mergerequests.create( + { + 'source_branch': branch.name, + 'target_project_id': 36527, # fdroid/fdroidserver + 'target_branch': 'master', + 'title': 'update to gradle v' + version, + 'description': description, + 'labels': ['fdroid-bot', 'buildserver'], + 'remove_source_branch': True, + } + ) + mr.save()