mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-05 15:00:11 +01:00
Merge branch 'gradle-fdroid-bot' into 'master'
fdroid-bot creates merge requests for gradle updates See merge request fdroid/fdroidserver!859
This commit is contained in:
Hans-Christoph Steiner
commit
2efab0f8ec
@ -226,21 +226,27 @@ fedora_latest:
|
||||
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
|
||||
|
||||
gradle:
|
||||
image: alpine:3.7
|
||||
variables:
|
||||
LANG: C.UTF-8
|
||||
image: debian:bullseye
|
||||
<<: *apt-template
|
||||
script:
|
||||
- apk add --no-cache ca-certificates git python3
|
||||
# if this is a merge request fork, then only check if makebuildserver changed
|
||||
- apt-get install
|
||||
ca-certificates
|
||||
git
|
||||
openssh-client
|
||||
python3-bs4
|
||||
python3-colorama
|
||||
python3-git
|
||||
python3-gitlab
|
||||
python3-requests
|
||||
# if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed
|
||||
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
|
||||
git fetch https://gitlab.com/fdroid/fdroidserver.git;
|
||||
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
|
||||
test "$f" == "makebuildserver" && export CHANGED="yes";
|
||||
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
|
||||
done;
|
||||
test -z "$CHANGED" && exit;
|
||||
fi
|
||||
- python3 -m ensurepip
|
||||
- $pip install beautifulsoup4 requests
|
||||
- ./tests/gradle-release-checksums.py
|
||||
|
||||
fdroid build:
|
||||
|
||||
@ -57,7 +57,17 @@ download_gradle() {
|
||||
|
||||
get_sha() {
|
||||
case $1 in
|
||||
'0.7') echo '4e354fcb0d5c0b0e7789cd6ee900456edaf993f6dd890c4a1c217d90d2a6a6ad' ;;
|
||||
'0.8') echo '940e623ea98e40ea9ad398770a6ebb91a61c0869d394dda81aa86b0f4f0025e7' ;;
|
||||
'0.9') echo '994e46d4b467254a0f25ce92b602618331b9b3ac8b32a094fd84ff0e0ceec135' ;;
|
||||
'0.9.1') echo '5d48cba95db031ec109ae9ab60561e960b6507046036e8191aa78572ec27e2a5' ;;
|
||||
'0.9.2') echo 'f94d7642348c558fc05ab5fd6fb947fb1ed8fed5931ddb73dd04fb0de22d669b' ;;
|
||||
'1.0') echo '894bca0360a7e2040815096788f118a2dd106ff6694221b274efb9c32bce0384' ;;
|
||||
'1.1') echo '552c1fc9f3a1b9668b79cc447370f0263e664ffb6d5c6e1c21e77ce0c8a20d4c' ;;
|
||||
'1.2') echo 'eb53da3704d24cabb7565f34a3bf16bcd863c4b0c139917606fb15d4f27c7bdf' ;;
|
||||
'1.3') echo 'ada68561efbb9f1cae0f9063974cbde15c180351a2f92bc2f1106e39ddcae5ba' ;;
|
||||
'1.4') echo 'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e' ;;
|
||||
'1.5') echo 'a5511a0659caa47d9d74fd2844c9da43157d2f78e63a0223c6289d88f5aaecbe' ;;
|
||||
'1.6') echo 'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4' ;;
|
||||
'1.7') echo '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30' ;;
|
||||
'1.8') echo 'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703' ;;
|
||||
@ -65,6 +75,7 @@ get_sha() {
|
||||
'1.10') echo '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe' ;;
|
||||
'1.11') echo '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091' ;;
|
||||
'1.12') echo '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea' ;;
|
||||
'2.0') echo 'a1eb880c8755333c4d33c4351b269bebe517002532d3142c0b6164c9e8c081c3' ;;
|
||||
'2.1') echo '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878' ;;
|
||||
'2.2') echo '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb' ;;
|
||||
'2.2.1') echo '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418' ;;
|
||||
|
||||
@ -1,58 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import git
|
||||
import gitlab
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
import sys
|
||||
import subprocess
|
||||
from bs4 import BeautifulSoup
|
||||
from colorama import Fore, Style
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
while True:
|
||||
r = requests.get('https://gradle.org/release-checksums/')
|
||||
if r.status_code == 200:
|
||||
break
|
||||
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
|
||||
version_pat = re.compile(r'[0-9]+(\.[0-9]+)+')
|
||||
|
||||
checksums = None
|
||||
versions = dict()
|
||||
for a in soup.find_all('a'):
|
||||
if a.parent.name != 'p':
|
||||
continue
|
||||
name = a.get('name')
|
||||
if not name:
|
||||
continue
|
||||
m = version_pat.search(name)
|
||||
|
||||
while not checksums:
|
||||
r = requests.get('https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json')
|
||||
if r.status_code == 200:
|
||||
checksums = r.json()
|
||||
|
||||
gradle_bin_pat = re.compile(r'gradle-([0-9][0-9.]+[0-9])-bin.zip')
|
||||
for url, d in checksums.items():
|
||||
m = gradle_bin_pat.search(url)
|
||||
if m:
|
||||
ul = a.parent.find_next_sibling('ul')
|
||||
versions[m.group()] = a.parent.find_next_sibling('ul').find('li').find('code').text.strip()
|
||||
versions[m.group(1)] = d[0]['sha256']
|
||||
|
||||
errors = 0
|
||||
makebuildserver = os.path.join(os.path.dirname(__file__), os.pardir, 'makebuildserver')
|
||||
with open(makebuildserver) as fp:
|
||||
contents = fp.read()
|
||||
to_compile = re.search(r'CACHE_FILES = [^\]]+\]', contents, flags=re.DOTALL | re.MULTILINE).group()
|
||||
makebuildserver_current = fp.read()
|
||||
to_compile = re.search(r'CACHE_FILES = [^\]]+\]', makebuildserver_current).group()
|
||||
code = compile(to_compile, makebuildserver, 'exec')
|
||||
config = {}
|
||||
exec(code, None, config) # nosec this is just a CI script
|
||||
makebuildserver_versions = []
|
||||
version_pat = re.compile(r'[0-9]+(\.[0-9]+)+')
|
||||
for url, checksum in config['CACHE_FILES']:
|
||||
if 'gradle.org' in url:
|
||||
m = version_pat.search(url.split('/')[-1])
|
||||
if m:
|
||||
makebuildserver_versions.append(m.group())
|
||||
if checksum != versions[m.group()]:
|
||||
print('ERROR: checksum mismatch:', checksum, versions[m.group()])
|
||||
print(Fore.RED
|
||||
+ 'ERROR: checksum mismatch:', checksum, versions[m.group()]
|
||||
+ Style.RESET_ALL)
|
||||
errors += 1
|
||||
|
||||
# error if makebuildserver is missing the latest version
|
||||
for version in sorted(versions.keys()):
|
||||
if version not in makebuildserver_versions \
|
||||
and LooseVersion(version) > LooseVersion(sorted(makebuildserver_versions)[-1]):
|
||||
errors += 1
|
||||
print(" ('https://services.gradle.org/distributions/gradle-" + version + "-bin.zip',\n"
|
||||
" '" + versions[version] + "'),")
|
||||
add_before = """ ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',"""
|
||||
new = to_compile.replace(
|
||||
add_before,
|
||||
" ('https://services.gradle.org/distributions/gradle-" + version + "-bin.zip',\n"
|
||||
" '" + versions[version] + "'),\n" + add_before
|
||||
)
|
||||
makebuildserver_current = makebuildserver_current.replace(to_compile, new)
|
||||
|
||||
print('makebuildserver has gradle v' + sorted(makebuildserver_versions)[-1])
|
||||
sys.exit(errors)
|
||||
with open('makebuildserver', 'w') as fp:
|
||||
fp.write(makebuildserver_current)
|
||||
|
||||
# write out update to gradlew-fdroid
|
||||
with open('gradlew-fdroid') as fp:
|
||||
gradlew_fdroid = fp.read()
|
||||
current = ''
|
||||
get_sha_pat = re.compile(r""" +'([0-9][0-9.]+[0-9])'\)\s+echo '([0-9a-f]{64})' ;;\n""")
|
||||
for m in get_sha_pat.finditer(gradlew_fdroid):
|
||||
current += m.group()
|
||||
checksum = m.group(2)
|
||||
if checksum != versions[m.group(1)]:
|
||||
print(Fore.RED
|
||||
+ 'ERROR: checksum mismatch:', checksum, versions[m.group(1)]
|
||||
+ Style.RESET_ALL)
|
||||
errors += 1
|
||||
new = ''
|
||||
for version in sorted(versions.keys(), key=LooseVersion):
|
||||
sha256 = versions[version]
|
||||
spaces = ''
|
||||
for i in range(6 - len(version)):
|
||||
spaces += ' '
|
||||
new += """ '%s')%s echo '%s' ;;\n""" % (version, spaces, sha256)
|
||||
gradlew_fdroid = gradlew_fdroid.replace(current, new)
|
||||
plugin_v = ' '.join(sorted(versions.keys(), key=LooseVersion, reverse=True))
|
||||
plugin_v_pat = re.compile(r'\nplugin_v=\(([0-9. ]+)\)')
|
||||
with open('gradlew-fdroid', 'w') as fp:
|
||||
fp.write(plugin_v_pat.sub('\nplugin_v=(%s)' % plugin_v, gradlew_fdroid))
|
||||
|
||||
if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid-bot':
|
||||
p = subprocess.run(['git', '--no-pager', 'diff'])
|
||||
print(p.stdout)
|
||||
exit(errors)
|
||||
|
||||
git_repo = git.repo.Repo('.')
|
||||
modified = git_repo.git().ls_files(modified=True).split()
|
||||
if (git_repo.is_dirty()
|
||||
and ('gradlew-fdroid' in modified or 'makebuildserver' in modified)):
|
||||
branch = git_repo.create_head(os.path.basename(__file__), force=True)
|
||||
branch.checkout()
|
||||
git_repo.index.add(['gradlew-fdroid', 'makebuildserver'])
|
||||
author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
|
||||
git_repo.index.commit('gradle v' + version, author=author)
|
||||
project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
|
||||
url = ('https://gitlab-ci-token:%s@%s/%s.git'
|
||||
% (os.getenv('PERSONAL_ACCESS_TOKEN'), os.getenv('CI_SERVER_HOST'), project_path))
|
||||
remote_name = 'fdroid-bot'
|
||||
try:
|
||||
remote = git_repo.create_remote(remote_name, url)
|
||||
except git.exc.GitCommandError:
|
||||
remote = git.remote.Remote(git_repo, remote_name)
|
||||
remote.set_url(url)
|
||||
remote.push(force=True)
|
||||
git.remote.Remote.rm(git_repo, remote_name)
|
||||
|
||||
private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
|
||||
if not private_token:
|
||||
print(Fore.RED
|
||||
+ 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
|
||||
+ Style.RESET_ALL)
|
||||
sys.exit(1)
|
||||
gl = gitlab.Gitlab(os.getenv('CI_SERVER_URL'), api_version=4,
|
||||
private_token=private_token)
|
||||
project = gl.projects.get(project_path, lazy=True)
|
||||
description = (
|
||||
'see <https://gitlab.com/fdroid/gradle-transparency-log/-/blob/master/checksums.json>'
|
||||
'\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
|
||||
% (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
|
||||
)
|
||||
mr = project.mergerequests.create({
|
||||
'source_branch': branch.name,
|
||||
'target_project_id': 36527, # fdroid/fdroidserver
|
||||
'target_branch': 'master',
|
||||
'title': 'update to gradle v' + version,
|
||||
'description': description,
|
||||
'labels': ['fdroid-bot', 'gradle'],
|
||||
'remove_source_branch': True,
|
||||
})
|
||||
mr.save()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user