gitlab-ci: auto-generate merge request when NDK release found

Following the pattern of the gradle bot, this will check the transparency
log for any new NDK release.  If there are any, it will make a merge
request from @fdroid-bot.

.gitlab-ci.yml

@@ -261,7 +261,7 @@ fedora_latest:
         "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
 
 
-gradle:
+gradle/ndk:
   image: debian:bullseye
   <<: *apt-template
   variables:
@@ -276,16 +276,18 @@ gradle:
           python3-git
           python3-gitlab
           python3-requests
-    # if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed
+    # if this is a merge request fork, then only check if relevant files changed
     - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
         git fetch https://gitlab.com/fdroid/fdroidserver.git;
         for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
            test "$f" == "makebuildserver" && export CHANGED="yes";
            test "$f" == "gradlew-fdroid" && export CHANGED="yes";
+           test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
         done;
         test -z "$CHANGED" && exit;
       fi
     - ./tests/gradle-release-checksums.py
+    - ./tests/ndk-release-checksums.py
 
 
 # Run an actual build in a simple, faked version of the buildserver guest VM.

tests/ndk-release-checksums.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import git
+import gitlab
+import json
+import os
+import re
+import requests
+import subprocess
+from colorama import Fore, Style
+
+
+checksums = None
+versions = dict()
+
+while not checksums:
+    r = requests.get(
+        'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json'
+    )
+    if r.status_code == 200:
+        checksums = r.json()
+
+with open('fdroidserver/common.py') as fp:
+    common_py = fp.read()
+
+to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group()
+if not to_compile:
+    exit(1)
+code = compile(to_compile, '<string>', 'exec')
+config = {}
+exec(code, None, config)  # nosec this is just a CI script
+
+ndks = []
+errors = 0
+release = None
+revision = None
+for k, entries in checksums.items():
+    if k.startswith('https://dl.google.com/android/repository/android-ndk'):
+        m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k)
+        if m:
+            for entry in entries:
+                if 'source.properties' in entry:
+                    n = re.search(
+                        r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties']
+                    )
+                    if n:
+                        release = m.group(1)
+                        revision = n.group()
+                        ndks.append(
+                            {
+                                'url': k,
+                                'release': release,
+                                'revision': revision,
+                                'sha256': checksums[k][0]['sha256'],
+                            }
+                        )
+            for d in config['NDKS']:
+                if k == d['url']:
+                    sha256 = d['sha256']
+                    found = False
+                    for entry in entries:
+                        if sha256 == entry['sha256']:
+                            found = True
+                    if not found:
+                        print(
+                            Fore.RED
+                            + (
+                                'ERROR: checksum mismatch: %s != %s'
+                                % (sha256, entry['sha256'])
+                            )
+                            + Style.RESET_ALL
+                        )
+                        errors += 1
+
+with open('fdroidserver/common.py', 'w') as fp:
+    fp.write(
+        common_py.replace(
+            to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True)
+        )
+    )
+
+if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid':
+    p = subprocess.run(['git', '--no-pager', 'diff'])
+    print(p.stdout)
+    exit(errors)
+
+
+# This only runs after commits are pushed to fdroid/fdroidserver
+git_repo = git.repo.Repo('.')
+modified = git_repo.git().ls_files(modified=True).split()
+if git_repo.is_dirty() and 'fdroidserver/common.py' in modified:
+    branch = git_repo.create_head(os.path.basename(__file__), force=True)
+    branch.checkout()
+    git_repo.index.add(['fdroidserver/common.py'])
+    author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
+    git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author)
+    project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
+    url = 'https://gitlab-ci-token:%s@%s/%s.git' % (
+        os.getenv('PERSONAL_ACCESS_TOKEN'),
+        os.getenv('CI_SERVER_HOST'),
+        project_path,
+    )
+    remote_name = 'fdroid-bot'
+    try:
+        remote = git_repo.create_remote(remote_name, url)
+    except git.exc.GitCommandError:
+        remote = git.remote.Remote(git_repo, remote_name)
+        remote.set_url(url)
+    remote.push(force=True)
+    git.remote.Remote.rm(git_repo, remote_name)
+
+    private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
+    if not private_token:
+        print(
+            Fore.RED
+            + 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
+            + Style.RESET_ALL
+        )
+        exit(1)
+    gl = gitlab.Gitlab(
+        os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token
+    )
+    project = gl.projects.get(project_path, lazy=True)
+    description = (
+        'see <https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json>'
+        '\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
+        % (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
+    )
+    mr = project.mergerequests.create(
+        {
+            'source_branch': branch.name,
+            'target_project_id': 36527,  # fdroid/fdroidserver
+            'target_branch': 'master',
+            'title': 'update to gradle v' + version,
+            'description': description,
+            'labels': ['fdroid-bot', 'buildserver'],
+            'remove_source_branch': True,
+        }
+    )
+    mr.save()