fdroid-bot creates merge request when new gradle versions are found

.gitlab-ci.yml

@@ -226,11 +226,18 @@ fedora_latest:
         "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
 
 gradle:
-  image: alpine:3.7
+  image: debian:bullseye
-  variables:
+  <<: *apt-template
-    LANG: C.UTF-8
   script:
-    - apk add --no-cache ca-certificates git python3
+    - apt-get install
+          ca-certificates
+          git
+          openssh-client
+          python3-bs4
+          python3-colorama
+          python3-git
+          python3-gitlab
+          python3-requests
     # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed
     - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
         git fetch https://gitlab.com/fdroid/fdroidserver.git;
@@ -240,8 +247,6 @@ gradle:
         done;
         test -z "$CHANGED" && exit;
       fi
-    - python3 -m ensurepip
-    - $pip install beautifulsoup4 requests
     - ./tests/gradle-release-checksums.py
 
 fdroid build:

tests/gradle-release-checksums.py

@@ -1,32 +1,29 @@
 #!/usr/bin/env python3
 
+import git
+import gitlab
 import os
 import re
 import requests
-import sys
+import subprocess
 from bs4 import BeautifulSoup
+from colorama import Fore, Style
 from distutils.version import LooseVersion
 
-while True:
-    r = requests.get('https://gradle.org/release-checksums/')
-    if r.status_code == 200:
-        break
-
-soup = BeautifulSoup(r.text, 'html.parser')
-
-version_pat = re.compile(r'[0-9]+(\.[0-9]+)+')
 
+checksums = None
 versions = dict()
-for a in soup.find_all('a'):
+
-    if a.parent.name != 'p':
+while not checksums:
-        continue
+   r = requests.get('https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json')
-    name = a.get('name')
+   if r.status_code == 200:
-    if not name:
+       checksums = r.json()
-        continue
+
-    m = version_pat.search(name)
+gradle_bin_pat = re.compile(r'gradle-([0-9][0-9.]+[0-9])-bin.zip')
+for url, d in checksums.items():
+    m = gradle_bin_pat.search(url)
     if m:
-        ul = a.parent.find_next_sibling('ul')
+        versions[m.group(1)] = d[0]['sha256']
-        versions[m.group()] = a.parent.find_next_sibling('ul').find('li').find('code').text.strip()
 
 errors = 0
 makebuildserver = os.path.join(os.path.dirname(__file__), os.pardir, 'makebuildserver')
@@ -37,13 +34,16 @@ code = compile(to_compile, makebuildserver, 'exec')
 config = {}
 exec(code, None, config)  # nosec this is just a CI script
 makebuildserver_versions = []
+version_pat = re.compile(r'[0-9]+(\.[0-9]+)+')
 for url, checksum in config['CACHE_FILES']:
     if 'gradle.org' in url:
         m = version_pat.search(url.split('/')[-1])
         if m:
             makebuildserver_versions.append(m.group())
             if checksum != versions[m.group()]:
-                print('ERROR: checksum mismatch:', checksum, versions[m.group()])
+                print(Fore.RED
+                      + 'ERROR: checksum mismatch:', checksum, versions[m.group()]
+                      + Style.RESET_ALL)
                 errors += 1
 
 # error if makebuildserver is missing the latest version
@@ -68,6 +68,12 @@ current = ''
 get_sha_pat = re.compile(r""" +'([0-9][0-9.]+[0-9])'\)\s+echo '([0-9a-f]{64})' ;;\n""")
 for m in get_sha_pat.finditer(gradlew_fdroid):
     current += m.group()
+    checksum = m.group(2)
+    if checksum != versions[m.group(1)]:
+        print(Fore.RED
+              + 'ERROR: checksum mismatch:', checksum, versions[m.group(1)]
+              + Style.RESET_ALL)
+        errors += 1
 new = ''
 for version in sorted(versions.keys(), key=LooseVersion):
     sha256 = versions[version]
@@ -81,5 +87,53 @@ plugin_v_pat = re.compile(r'\nplugin_v=\(([0-9. ]+)\)')
 with open('gradlew-fdroid', 'w') as fp:
     fp.write(plugin_v_pat.sub('\nplugin_v=(%s)' % plugin_v, gradlew_fdroid))
 
-print('makebuildserver has gradle v' + sorted(makebuildserver_versions)[-1])
+if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid-bot':
-sys.exit(errors)
+    p = subprocess.run(['git', '--no-pager', 'diff'])
+    print(p.stdout)
+    exit(errors)
+
+git_repo = git.repo.Repo('.')
+modified = git_repo.git().ls_files(modified=True).split()
+if (git_repo.is_dirty()
+    and ('gradlew-fdroid' in modified or 'makebuildserver' in modified)):
+    branch = git_repo.create_head(os.path.basename(__file__), force=True)
+    branch.checkout()
+    git_repo.index.add(['gradlew-fdroid', 'makebuildserver'])
+    author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
+    git_repo.index.commit('gradle v' + version, author=author)
+    project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
+    url = ('https://gitlab-ci-token:%s@%s/%s.git'
+           % (os.getenv('PERSONAL_ACCESS_TOKEN'), os.getenv('CI_SERVER_HOST'), project_path))
+    remote_name = 'fdroid-bot'
+    try:
+        remote = git_repo.create_remote(remote_name, url)
+    except git.exc.GitCommandError:
+        remote = git.remote.Remote(git_repo, remote_name)
+        remote.set_url(url)
+    remote.push(force=True)
+    git.remote.Remote.rm(git_repo, remote_name)
+
+    private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
+    if not private_token:
+        print(Fore.RED
+              + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
+              + Style.RESET_ALL)
+        sys.exit(1)
+    gl = gitlab.Gitlab(os.getenv('CI_SERVER_URL'), api_version=4,
+                       private_token=private_token)
+    project = gl.projects.get(project_path, lazy=True)
+    description = (
+        'see <https://gitlab.com/fdroid/gradle-transparency-log/-/blob/master/checksums.json>'
+        '\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
+        % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
+    )
+    mr = project.mergerequests.create({
+        'source_branch': branch.name,
+        'target_project_id': 36527,  # fdroid/fdroidserver
+        'target_branch': 'master',
+        'title': 'update to gradle v' + version,
+        'description': description,
+        'labels': ['fdroid-bot', 'gradle'],
+        'remove_source_branch': True,
+    })
+    mr.save()