From eec0d3a336f8494f9d756d23dd646df79de8fb50 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 24 Jan 2021 11:57:46 +0100 Subject: [PATCH 1/2] gitlab-ci: gradle job generates required changes for new versions --- .gitlab-ci.yml | 3 ++- gradlew-fdroid | 11 +++++++++ tests/gradle-release-checksums.py | 37 ++++++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26c900ad..4797cef8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -231,11 +231,12 @@ gradle: LANG: C.UTF-8 script: - apk add --no-cache ca-certificates git python3 - # if this is a merge request fork, then only check if makebuildserver changed + # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroidserver.git; for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do test "$f" == "makebuildserver" && export CHANGED="yes"; + test "$f" == "gradlew-fdroid" && export CHANGED="yes"; done; test -z "$CHANGED" && exit; fi diff --git a/gradlew-fdroid b/gradlew-fdroid index ea34397f..53494da2 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -57,7 +57,17 @@ download_gradle() { get_sha() { case $1 in + '0.7') echo '4e354fcb0d5c0b0e7789cd6ee900456edaf993f6dd890c4a1c217d90d2a6a6ad' ;; + '0.8') echo '940e623ea98e40ea9ad398770a6ebb91a61c0869d394dda81aa86b0f4f0025e7' ;; + '0.9') echo '994e46d4b467254a0f25ce92b602618331b9b3ac8b32a094fd84ff0e0ceec135' ;; + '0.9.1') echo '5d48cba95db031ec109ae9ab60561e960b6507046036e8191aa78572ec27e2a5' ;; + '0.9.2') echo 'f94d7642348c558fc05ab5fd6fb947fb1ed8fed5931ddb73dd04fb0de22d669b' ;; + '1.0') echo '894bca0360a7e2040815096788f118a2dd106ff6694221b274efb9c32bce0384' ;; + '1.1') echo '552c1fc9f3a1b9668b79cc447370f0263e664ffb6d5c6e1c21e77ce0c8a20d4c' ;; + '1.2') echo 'eb53da3704d24cabb7565f34a3bf16bcd863c4b0c139917606fb15d4f27c7bdf' ;; + '1.3') echo 'ada68561efbb9f1cae0f9063974cbde15c180351a2f92bc2f1106e39ddcae5ba' ;; '1.4') echo 'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e' ;; + '1.5') echo 'a5511a0659caa47d9d74fd2844c9da43157d2f78e63a0223c6289d88f5aaecbe' ;; '1.6') echo 'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4' ;; '1.7') echo '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30' ;; '1.8') echo 'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703' ;; @@ -65,6 +75,7 @@ get_sha() { '1.10') echo '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe' ;; '1.11') echo '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091' ;; '1.12') echo '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea' ;; + '2.0') echo 'a1eb880c8755333c4d33c4351b269bebe517002532d3142c0b6164c9e8c081c3' ;; '2.1') echo '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878' ;; '2.2') echo '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb' ;; '2.2.1') echo '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418' ;; diff --git a/tests/gradle-release-checksums.py b/tests/gradle-release-checksums.py index 165c56da..ce2c79cf 100755 --- a/tests/gradle-release-checksums.py +++ b/tests/gradle-release-checksums.py @@ -31,8 +31,8 @@ for a in soup.find_all('a'): errors = 0 makebuildserver = os.path.join(os.path.dirname(__file__), os.pardir, 'makebuildserver') with open(makebuildserver) as fp: - contents = fp.read() -to_compile = re.search(r'CACHE_FILES = [^\]]+\]', contents, flags=re.DOTALL | re.MULTILINE).group() + makebuildserver_current = fp.read() +to_compile = re.search(r'CACHE_FILES = [^\]]+\]', makebuildserver_current).group() code = compile(to_compile, makebuildserver, 'exec') config = {} exec(code, None, config) # nosec this is just a CI script @@ -50,9 +50,36 @@ for url, checksum in config['CACHE_FILES']: for version in sorted(versions.keys()): if version not in makebuildserver_versions \ and LooseVersion(version) > LooseVersion(sorted(makebuildserver_versions)[-1]): - errors += 1 - print(" ('https://services.gradle.org/distributions/gradle-" + version + "-bin.zip',\n" - " '" + versions[version] + "'),") + add_before = """ ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',""" + new = to_compile.replace( + add_before, + " ('https://services.gradle.org/distributions/gradle-" + version + "-bin.zip',\n" + " '" + versions[version] + "'),\n" + add_before + ) + makebuildserver_current = makebuildserver_current.replace(to_compile, new) + +with open('makebuildserver', 'w') as fp: + fp.write(makebuildserver_current) + +# write out update to gradlew-fdroid +with open('gradlew-fdroid') as fp: + gradlew_fdroid = fp.read() +current = '' +get_sha_pat = re.compile(r""" +'([0-9][0-9.]+[0-9])'\)\s+echo '([0-9a-f]{64})' ;;\n""") +for m in get_sha_pat.finditer(gradlew_fdroid): + current += m.group() +new = '' +for version in sorted(versions.keys(), key=LooseVersion): + sha256 = versions[version] + spaces = '' + for i in range(6 - len(version)): + spaces += ' ' + new += """ '%s')%s echo '%s' ;;\n""" % (version, spaces, sha256) +gradlew_fdroid = gradlew_fdroid.replace(current, new) +plugin_v = ' '.join(sorted(versions.keys(), key=LooseVersion, reverse=True)) +plugin_v_pat = re.compile(r'\nplugin_v=\(([0-9. ]+)\)') +with open('gradlew-fdroid', 'w') as fp: + fp.write(plugin_v_pat.sub('\nplugin_v=(%s)' % plugin_v, gradlew_fdroid)) print('makebuildserver has gradle v' + sorted(makebuildserver_versions)[-1]) sys.exit(errors) From d133b38f223796415a0188042a7af9c43a6c7db8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 24 Jan 2021 17:09:32 +0100 Subject: [PATCH 2/2] fdroid-bot creates merge request when new gradle versions are found --- .gitlab-ci.yml | 17 ++++-- tests/gradle-release-checksums.py | 96 ++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4797cef8..011e9048 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -226,11 +226,18 @@ fedora_latest: "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" gradle: - image: alpine:3.7 - variables: - LANG: C.UTF-8 + image: debian:bullseye + <<: *apt-template script: - - apk add --no-cache ca-certificates git python3 + - apt-get install + ca-certificates + git + openssh-client + python3-bs4 + python3-colorama + python3-git + python3-gitlab + python3-requests # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroidserver.git; @@ -240,8 +247,6 @@ gradle: done; test -z "$CHANGED" && exit; fi - - python3 -m ensurepip - - $pip install beautifulsoup4 requests - ./tests/gradle-release-checksums.py fdroid build: diff --git a/tests/gradle-release-checksums.py b/tests/gradle-release-checksums.py index ce2c79cf..a264c8ba 100755 --- a/tests/gradle-release-checksums.py +++ b/tests/gradle-release-checksums.py @@ -1,32 +1,29 @@ #!/usr/bin/env python3 +import git +import gitlab import os import re import requests -import sys +import subprocess from bs4 import BeautifulSoup +from colorama import Fore, Style from distutils.version import LooseVersion -while True: - r = requests.get('https://gradle.org/release-checksums/') - if r.status_code == 200: - break - -soup = BeautifulSoup(r.text, 'html.parser') - -version_pat = re.compile(r'[0-9]+(\.[0-9]+)+') +checksums = None versions = dict() -for a in soup.find_all('a'): - if a.parent.name != 'p': - continue - name = a.get('name') - if not name: - continue - m = version_pat.search(name) + +while not checksums: + r = requests.get('https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json') + if r.status_code == 200: + checksums = r.json() + +gradle_bin_pat = re.compile(r'gradle-([0-9][0-9.]+[0-9])-bin.zip') +for url, d in checksums.items(): + m = gradle_bin_pat.search(url) if m: - ul = a.parent.find_next_sibling('ul') - versions[m.group()] = a.parent.find_next_sibling('ul').find('li').find('code').text.strip() + versions[m.group(1)] = d[0]['sha256'] errors = 0 makebuildserver = os.path.join(os.path.dirname(__file__), os.pardir, 'makebuildserver') @@ -37,13 +34,16 @@ code = compile(to_compile, makebuildserver, 'exec') config = {} exec(code, None, config) # nosec this is just a CI script makebuildserver_versions = [] +version_pat = re.compile(r'[0-9]+(\.[0-9]+)+') for url, checksum in config['CACHE_FILES']: if 'gradle.org' in url: m = version_pat.search(url.split('/')[-1]) if m: makebuildserver_versions.append(m.group()) if checksum != versions[m.group()]: - print('ERROR: checksum mismatch:', checksum, versions[m.group()]) + print(Fore.RED + + 'ERROR: checksum mismatch:', checksum, versions[m.group()] + + Style.RESET_ALL) errors += 1 # error if makebuildserver is missing the latest version @@ -68,6 +68,12 @@ current = '' get_sha_pat = re.compile(r""" +'([0-9][0-9.]+[0-9])'\)\s+echo '([0-9a-f]{64})' ;;\n""") for m in get_sha_pat.finditer(gradlew_fdroid): current += m.group() + checksum = m.group(2) + if checksum != versions[m.group(1)]: + print(Fore.RED + + 'ERROR: checksum mismatch:', checksum, versions[m.group(1)] + + Style.RESET_ALL) + errors += 1 new = '' for version in sorted(versions.keys(), key=LooseVersion): sha256 = versions[version] @@ -81,5 +87,53 @@ plugin_v_pat = re.compile(r'\nplugin_v=\(([0-9. ]+)\)') with open('gradlew-fdroid', 'w') as fp: fp.write(plugin_v_pat.sub('\nplugin_v=(%s)' % plugin_v, gradlew_fdroid)) -print('makebuildserver has gradle v' + sorted(makebuildserver_versions)[-1]) -sys.exit(errors) +if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid-bot': + p = subprocess.run(['git', '--no-pager', 'diff']) + print(p.stdout) + exit(errors) + +git_repo = git.repo.Repo('.') +modified = git_repo.git().ls_files(modified=True).split() +if (git_repo.is_dirty() + and ('gradlew-fdroid' in modified or 'makebuildserver' in modified)): + branch = git_repo.create_head(os.path.basename(__file__), force=True) + branch.checkout() + git_repo.index.add(['gradlew-fdroid', 'makebuildserver']) + author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org') + git_repo.index.commit('gradle v' + version, author=author) + project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME') + url = ('https://gitlab-ci-token:%s@%s/%s.git' + % (os.getenv('PERSONAL_ACCESS_TOKEN'), os.getenv('CI_SERVER_HOST'), project_path)) + remote_name = 'fdroid-bot' + try: + remote = git_repo.create_remote(remote_name, url) + except git.exc.GitCommandError: + remote = git.remote.Remote(git_repo, remote_name) + remote.set_url(url) + remote.push(force=True) + git.remote.Remote.rm(git_repo, remote_name) + + private_token = os.getenv('PERSONAL_ACCESS_TOKEN') + if not private_token: + print(Fore.RED + + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!' + + Style.RESET_ALL) + sys.exit(1) + gl = gitlab.Gitlab(os.getenv('CI_SERVER_URL'), api_version=4, + private_token=private_token) + project = gl.projects.get(project_path, lazy=True) + description = ( + 'see ' + '\n\n

generated by GitLab CI Job #%s

' + % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID')) + ) + mr = project.mergerequests.create({ + 'source_branch': branch.name, + 'target_project_id': 36527, # fdroid/fdroidserver + 'target_branch': 'master', + 'title': 'update to gradle v' + version, + 'description': description, + 'labels': ['fdroid-bot', 'gradle'], + 'remove_source_branch': True, + }) + mr.save()