Merge branch 'index-v2-nightly' into 'master'

nightly bug fixes and tests

See merge request fdroid/fdroidserver!1257

.gitlab-ci.yml

@@ -148,11 +148,12 @@ ubuntu_jammy_pip:
     # back to bare machine to act as user's install machine
     - export ANDROID_HOME=/opt/android-sdk
     - $pip install sdkmanager
-    - sdkmanager 'build-tools;30.0.0'
+    - sdkmanager 'build-tools;33.0.0'
 
     - $pip install dist/fdroidserver-*.tar.gz
     - tar xzf dist/fdroidserver-*.tar.gz
     - cd fdroidserver-*
+    - export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0
     - fdroid=`which fdroid` ./tests/run-tests
 
 
@@ -246,6 +247,7 @@ black:
         tests/lint.TestCase
         tests/metadata.TestCase
         tests/ndk-release-checksums.py
+        tests/nightly.TestCase
         tests/rewritemeta.TestCase
         tests/scanner.TestCase
         tests/signindex.TestCase

MANIFEST.in

@@ -42,6 +42,7 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po
 include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
 include makebuildserver
 include README.md
+include tests/aosp_testkey_debug.keystore
 include tests/apk.embedded_1.apk
 include tests/bad-unicode-*.apk
 include tests/build.TestCase
@@ -623,6 +624,7 @@ include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml
 include tests/metadata/souch.smsbypass.yml
 include tests/metadata.TestCase
 include tests/minimal_targetsdk_30_unsigned.apk
+include tests/nightly.TestCase
 include tests/Norway_bouvet_europe_2.obf.zip
 include tests/no_targetsdk_minsdk1_unsigned.apk
 include tests/no_targetsdk_minsdk30_unsigned.apk

fdroidserver/common.py

@@ -2680,9 +2680,9 @@ def get_native_code(apkfile):
 
 
 class PopenResult:
-    def __init__(self):
-        self.returncode = None
-        self.output = None
+    def __init__(self, returncode=None, output=None):
+        self.returncode = returncode
+        self.output = output
 
 
 def SdkToolsPopen(commands, cwd=None, output=True):

fdroidserver/deploy.py

@@ -25,6 +25,7 @@ import re
 import subprocess
 import time
 import urllib
+import yaml
 from argparse import ArgumentParser
 import logging
 import shutil
@@ -376,7 +377,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
     if repo_section == 'repo':
         git_mirror_path = 'git-mirror'
         dotgit = os.path.join(git_mirror_path, '.git')
-        git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section)
+        git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
+        git_repodir = os.path.join(git_fdroiddir, repo_section)
         if not os.path.isdir(git_repodir):
             os.makedirs(git_repodir)
         # github/gitlab use bare git repos, so only count the .git folder
@@ -438,6 +440,18 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         else:
             progress = None
 
+        # only deploy to GitLab Artifacts if too big for GitLab Pages
+        if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+            gitlab_ci_job_name = 'pages'
+        else:
+            gitlab_ci_job_name = 'GitLab Artifacts'
+            logging.warning(
+                _(
+                    'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
+                )
+                % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
+            )
+
         # push for every remote. This will overwrite the git history
         for remote in repo.remotes:
             if remote.name not in enabled_remotes:
@@ -445,16 +459,22 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                 continue
             if remote.name == 'gitlab':
                 logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
-                with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file:
-                    out_file.write("""pages:
-  script:
-   - mkdir .public
-   - cp -r * .public/
-   - mv .public public
-  artifacts:
-    paths:
-    - public
-""")
+                with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
+                    yaml.dump(
+                        {
+                            gitlab_ci_job_name: {
+                                'script': [
+                                    'mkdir .public',
+                                    'cp -r * .public/',
+                                    'mv .public public',
+                                ],
+                                'artifacts': {'paths': ['public']},
+                                'variables': {'GIT_DEPTH': 1},
+                            }
+                        },
+                        fp,
+                        default_flow_style=False,
+                    )
 
                 repo.git.add(all=True)
                 repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")

fdroidserver/index.py

@@ -1438,7 +1438,8 @@ def get_mirror_service_urls(url):
         segments.extend([branch, folder])
         urls.append('/'.join(segments))
     elif hostname == "gitlab.com":
-        if common.get_dir_size(folder) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+        git_mirror_path = os.path.join('git-mirror', folder)
+        if common.get_dir_size(git_mirror_path) <= common.GITLAB_COM_PAGES_MAX_SIZE:
             # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
             gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
             urls.append('/'.join(gitlab_pages))
@@ -1452,6 +1453,26 @@ def get_mirror_service_urls(url):
         # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
         gitlab_raw = segments + ['-', 'raw', branch, folder]
         urls.append('/'.join(gitlab_raw))
+        # GitLab Artifacts "https://user.gitlab.io/-/repo/-/jobs/job_id/artifacts/public/folder"
+        job_id = os.getenv('CI_JOB_ID')
+        try:
+            int(job_id)
+            gitlab_artifacts = [
+                "https:",
+                "",
+                user + ".gitlab.io",
+                '-',
+                repo,
+                '-',
+                'jobs',
+                job_id,
+                'artifacts',
+                'public',
+                folder,
+            ]
+            urls.append('/'.join(gitlab_artifacts))
+        except (TypeError, ValueError):
+            pass  # no Job ID to use, ignore
 
     return urls
 

fdroidserver/nightly.py

@@ -25,6 +25,7 @@ import os
 import paramiko
 import platform
 import shutil
+import ssl
 import subprocess
 import sys
 import tempfile
@@ -34,7 +35,7 @@ from argparse import ArgumentParser
 
 from . import _
 from . import common
-
+from .exception import VCSException
 
 # hard coded defaults for Android ~/.android/debug.keystore files
 # https://developers.google.com/android/guides/client-auth
@@ -47,7 +48,16 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US'
 NIGHTLY = '-nightly'
 
 
-def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE):
+def _get_keystore_secret_var(keystore):
+    with open(keystore, 'rb') as fp:
+        return base64.standard_b64encode(fp.read()).decode('ascii')
+
+
+def _ssh_key_from_debug_keystore(keystore=None):
+    if keystore is None:
+        # set this here so it can be overridden in the tests
+        # TODO convert this to a class to get rid of this nonsense
+        keystore = KEYSTORE_FILE
     tmp_dir = tempfile.mkdtemp(prefix='.')
     privkey = os.path.join(tmp_dir, '.privkey')
     key_pem = os.path.join(tmp_dir, '.key.pem')
@@ -94,10 +104,17 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE):
         ],
         env={'LC_ALL': 'C.UTF-8'},
     )
+
+    # OpenSSL 3.0 changed the default output format from PKCS#1 to
+    # PKCS#8, which paramiko does not support.
+    # https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional
+    # https://github.com/paramiko/paramiko/issues/1015
+    openssl_rsa_cmd = ['openssl', 'rsa']
+    if ssl.OPENSSL_VERSION_INFO[0] >= 3:
+        openssl_rsa_cmd += ['-traditional']
     subprocess.check_call(
-        [
-            'openssl',
-            'rsa',
+        openssl_rsa_cmd
+        + [
             '-in',
             key_pem,
             '-out',
@@ -180,14 +197,16 @@ def main():
         default=False,
         help=_("Don't use rsync checksums"),
     )
+    archive_older_unset = -1
     parser.add_argument(
         "--archive-older",
         type=int,
-        default=20,
+        default=archive_older_unset,
         help=_("Set maximum releases in repo before older ones are archived"),
     )
     # TODO add --with-btlog
     options = parser.parse_args()
+    common.options = options
 
     # force a tighter umask since this writes private key material
     umask = os.umask(0o077)
@@ -262,15 +281,17 @@ def main():
 
         repo_url = repo_base + '/repo'
         git_mirror_path = os.path.join(repo_basedir, 'git-mirror')
-        git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo')
-        git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata')
-        git_mirror_statsdir = os.path.join(git_mirror_path, 'fdroid', 'stats')
+        git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
+        git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
+        git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
+        git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats')
         if not os.path.isdir(git_mirror_repodir):
             logging.debug(_('cloning {url}').format(url=clone_url))
-            try:
-                git.Repo.clone_from(clone_url, git_mirror_path)
-            except Exception:
-                pass
+            vcs = common.getvcs('git', clone_url, git_mirror_path)
+            p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)])
+            if p.returncode != 0:
+                print('WARNING: only public git repos are supported!')
+                raise VCSException('git clone %s failed:' % clone_url, p.output)
         if not os.path.isdir(git_mirror_repodir):
             os.makedirs(git_mirror_repodir, mode=0o755)
 
@@ -316,28 +337,40 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         with open(ssh_config, 'a') as fp:
             fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file)
 
-        config = ''
-        config += "identity_file = '%s'\n" % ssh_private_key_file
-        config += "repo_name = '%s'\n" % repo_git_base
-        config += "repo_url = '%s'\n" % repo_url
-        config += "repo_description = 'Nightly builds from %s'\n" % git_user_email
-        config += "archive_name = '%s'\n" % (repo_git_base + ' archive')
-        config += "archive_url = '%s'\n" % (repo_base + '/archive')
-        config += (
-            "archive_description = 'Old nightly builds that have been archived.'\n"
-        )
-        config += "archive_older = %i\n" % options.archive_older
-        config += "servergitmirrors = '%s'\n" % servergitmirror
-        config += "keystore = '%s'\n" % KEYSTORE_FILE
-        config += "repo_keyalias = '%s'\n" % KEY_ALIAS
-        config += "keystorepass = '%s'\n" % PASSWORD
-        config += "keypass = '%s'\n" % PASSWORD
-        config += "keydname = '%s'\n" % DISTINGUISHED_NAME
-        config += "make_current_version_link = False\n"
-        config += "update_stats = True\n"
-        with open('config.py', 'w') as fp:
-            fp.write(config)
-        os.chmod('config.py', 0o600)
+        if options.archive_older == archive_older_unset:
+            fdroid_size = common.get_dir_size(git_mirror_fdroiddir)
+            max_size = common.GITLAB_COM_PAGES_MAX_SIZE
+            if fdroid_size < max_size:
+                options.archive_older = 20
+            else:
+                options.archive_older = 3
+                print(
+                    'WARNING: repo is %s over the GitLab Pages limit (%s)'
+                    % (fdroid_size - max_size, max_size)
+                )
+                print('Setting --archive-older to 3')
+
+        config = {
+            'identity_file': ssh_private_key_file,
+            'repo_name': repo_git_base,
+            'repo_url': repo_url,
+            'repo_description': 'Nightly builds from %s' % git_user_email,
+            'archive_name': repo_git_base + ' archive',
+            'archive_url': repo_base + '/archive',
+            'archive_description': 'Old nightly builds that have been archived.',
+            'archive_older': options.archive_older,
+            'servergitmirrors': servergitmirror,
+            'keystore': KEYSTORE_FILE,
+            'repo_keyalias': KEY_ALIAS,
+            'keystorepass': PASSWORD,
+            'keypass': PASSWORD,
+            'keydname': DISTINGUISHED_NAME,
+            'make_current_version_link': False,
+            'update_stats': True,
+        }
+        with open('config.yml', 'w') as fp:
+            yaml.dump(config, fp, default_flow_style=False)
+        os.chmod('config.yml', 0o600)
         config = common.read_config(options)
         common.assert_config_keystore(config)
 
@@ -430,17 +463,16 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
                           + '\n     -dname "CN=Android Debug,O=Android,C=US"')
             sys.exit(1)
         ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
-        os.makedirs(os.path.dirname(ssh_dir), exist_ok=True)
         privkey = _ssh_key_from_debug_keystore(options.keystore)
-        ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
-        shutil.move(privkey, ssh_private_key_file)
-        shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
+        if os.path.exists(ssh_dir):
+            ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
+            shutil.move(privkey, ssh_private_key_file)
+            shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
         if shutil.rmtree.avoids_symlink_attacks:
             shutil.rmtree(os.path.dirname(privkey))
 
         if options.show_secret_var:
-            with open(options.keystore, 'rb') as fp:
-                debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii')
+            debug_keystore = _get_keystore_secret_var(options.keystore)
             print(
                 _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format(
                     path=options.keystore

tests/index.TestCase

@@ -414,10 +414,17 @@ class IndexTest(unittest.TestCase):
                 fdroidserver.index.get_mirror_service_urls(url),
             )
 
+    @patch.dict(os.environ, clear=True)
     def test_gitlab_get_mirror_service_urls(self):
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            os.mkdir('fdroid')
-            with Path('fdroid/placeholder').open('w') as fp:
+            git_mirror_path = Path('git-mirror/fdroid')
+            git_mirror_path.mkdir(parents=True)
+            ci_job_id = '12345678'
+            artifacts_url = (
+                'https://group.gitlab.io/-/project/-/jobs/%s/artifacts/public/fdroid'
+                % ci_job_id
+            )
+            with (git_mirror_path / 'placeholder').open('w') as fp:
                 fp.write('                                        ')
             for url in [
                 'git@gitlab.com:group/project',
@@ -426,20 +433,34 @@ class IndexTest(unittest.TestCase):
                 'https://gitlab.com/group/project.git',
             ]:
                 with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 1000):
+                    expected = [
+                        'https://group.gitlab.io/project/fdroid',
+                        'https://gitlab.com/group/project/-/raw/master/fdroid',
+                    ]
                     self.assertEqual(
-                        [
-                            'https://group.gitlab.io/project/fdroid',
-                            'https://gitlab.com/group/project/-/raw/master/fdroid',
-                        ],
+                        expected,
                         fdroidserver.index.get_mirror_service_urls(url),
                     )
+                    with patch.dict(os.environ, clear=True):
+                        os.environ['CI_JOB_ID'] = ci_job_id
+                        self.assertEqual(
+                            expected + [artifacts_url],
+                            fdroidserver.index.get_mirror_service_urls(url),
+                        )
                 with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10):
+                    expected = [
+                        'https://gitlab.com/group/project/-/raw/master/fdroid',
+                    ]
                     self.assertEqual(
-                        [
-                            'https://gitlab.com/group/project/-/raw/master/fdroid',
-                        ],
+                        expected,
                         fdroidserver.index.get_mirror_service_urls(url),
                     )
+                    with patch.dict(os.environ, clear=True):
+                        os.environ['CI_JOB_ID'] = ci_job_id
+                        self.assertEqual(
+                            expected + [artifacts_url],
+                            fdroidserver.index.get_mirror_service_urls(url),
+                        )
 
     def test_make_website(self):
         tmptestsdir = tempfile.mkdtemp(

tests/nightly.TestCase

@@ -1,11 +1,20 @@
 #!/usr/bin/env python3
 
 import inspect
+import logging
 import optparse
 import os
 import requests
+import shutil
+import subprocess
 import sys
+import tempfile
+import time
 import unittest
+import yaml
+
+from pathlib import Path
+from unittest.mock import patch
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
@@ -14,10 +23,67 @@ print('localmodule: ' + localmodule)
 if localmodule not in sys.path:
     sys.path.insert(0, localmodule)
 
-from fdroidserver import common, nightly
+from fdroidserver import common, exception, index, nightly
+
+
+DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABNYhAuskAAAK8MIICuDAOBgorBgEEASoCEQEBBQAEggKkqRnFlhidQmVff83bsAeewXPIsF0jiymzJnvrnUAQtCK0MV9uZonu37Mrj/qKLn56mf6QcvEoKvpCstZxzftgYYpAHWMVLM+hy2Z707QZEHlY7Ukppt8DItj+dXkeqGt7f8KzOb2AQwDbt9lm1fJb+MefLowTaubtvrLMcKIne43CbCu2D8HyN7RPWpEkVetA2Qgr5W4sa3tIUT80afqo9jzwJjKCspuxY9A1M8EIM3/kvyLo2B9r0cuWwRjYZXJ6gmTYI2ARNz0KQnCZUok14NDg+mZTb1B7AzRfb0lfjbA6grbzuAL+WaEpO8/LgGfuOh7QBZBT498TElOaFfQ9toQWA79wAmrQCm4OoFukpPIy2m/l6VjJSmlK5Q+CMOl/Au7OG1sUUCTvPaIr0XKnsiwDJ7a71n9garnPWHkvuWapSRCzCNgaUoGQjB+fTMJFFrwT8P1aLfM6onc3KNrDStoQZuYe5ngCLlNS56bENkVGvJBfdkboxtHZjqDXXON9jWGSOI527J3o2D5sjSVyx3T9XPrsL4TA/nBtdU+c/+M6aoASZR2VymzAKdMrGfj9kE5GXp8vv2vkJj9+OJ4Jm5yeczocc/Idtojjb1yg+sq1yY8kAQxgezpY1rpgi2jF3tSN01c23DNvAaSJLJX2ZuH8sD40ACc80Y1Qp1nUTdpwBZUeaeNruBwx4PHU8GnC71FwtiUpwNs0OoSl0pgDUJ3ODC5bs8B5QmW1wu1eg7I4mMSmCsNGW6VN3sFcu+WEqnmTxPoZombdFZKxsr2oq359Nn4bJ6Uc9PBz/sXsns7Zx1vND/oK/Jv5Y269UVAMeKX/eGpfnxzagW3tqGbOu12C2p9Azo5VxiU2fG/tmk2PjaG5hV/ywReco7I6C1p8OWM2fwAAAAEABVguNTA5AAAB6TCCAeUwggFOoAMCAQICBE89gTUwDQYJKoZIhvcNAQEFBQAwNzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0FuZHJvaWQxFjAUBgNVBAMTDUFuZHJvaWQgRGVidWcwHhcNMTIwMjE2MjIyMDM3WhcNNDIwMjA4MjIyMDM3WjA3MQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQW5kcm9pZDEWMBQGA1UEAxMNQW5kcm9pZCBEZWJ1ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA3AKU7S7JXhUjEwxWP1/LPHXieh61SaA/+xbpqsPA+yjGz1sAcGAyuG6bjNAVm56pq7nkjJzicX7Wi83nUBo58DEC/quxOLdy0C4PEOSAeTnTT1RJIwMDvOgiL1GFCErvQ7gCH6zuAID/JRFbN6nIkhDjs2DYnSBl7aJJf8wCLc0CAwEAATANBgkqhkiG9w0BAQUFAAOBgQAoq/TJffA0l+ZGf89xndmHdxrO6qi+TzSlByvLZ4eFfCovTh1iO+Edrd5V1yXGLxyyvdsadMAFZT8SaxMrP5xxhJ0nra0APWYLpA96M//auMhQBWPgqPntwgvEZuEH7f0kdItjBJ39yijbG8xfgwid6XqNUo0TDDkp/wNWKpJ9tJe+2PrGw1NAvrgSydoH2j8DI1Eq'
+DEBUG_KEYSTORE_KEY_FILE_NAME = (
+    'debug_keystore_QW+xRCJDGHXyyFtgCW8QRajj+6uYmsLwGWpCfYqYQ5M_id_rsa'
+)
+
+AOSP_TESTKEY_DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABejjuIU0AAAUBMIIE/TAOBgorBgEEASoCEQEBBQAEggTpvqhdBtq9D3jRUZGnhKLbFH1LMtCKqwGg25ETAEhvK1GVRNuWAHAUUedCnarjgeUy/zx9OsHuZq18KjUI115kWq/jxkf00fIg7wrOmXoyJf5Dbc7NGKjU64rRmppQEkJ417Lq4Uola9EBJ/WweEu6UTjTn5HcNl4mVloWKMBKNPkVfhZhAkXUyjiZ9rCVHMjLOVKG5vyTWZLwXpYR00Xz6VyzSunTyDza5oUOT/Fh7Gw74V7iNHANydkBHmH+UJ100p0vNPRFvt/3ABfMjkNbRXKNERnyN7NeBmCAOceuXjme/n0XLUidP9/NYk1yAmRJgUnauKD6UPSZYaUPuNSSdf4dD5fCQ7OVDq95e7vmqRDfrKUoWmtpndN7hbVl+OHVZXk2ngvXbvoS+F7ShsEfbq7+c37dnOcVrIlrY+wlOWX2jN42T+AkGt3AfA8zdIPdNgLGk64Op+aP4vGyLQqbuUEzOTNG9uExjGlamogPKFf93GAF83xv7AChYLR/9H+B1E955FL58bRuYOXVWJfLRsO/jyjXsilhBggo3VD1omRuOp98AkKP+P9JXCTswK7IZgvbMK3GB6QIzD20vlT0eK6JGLeWE7cXVn6oT26zvnqAjJ94PjS+YckMOExhqwCivPp1VaX6JzpQ1wr52OsGDUvconcjYrBEHBiY+UnMUk0Wj4mhZlJd1lpybZcWZ3vhTIlM0uMt4udl7t+zsgZ6BW97/pkGaa+QoxeTvgNlHGYyDYp8hveM3bCLXTHULw8mXUHxOJawq/J3E6vZ5/h2nzfmQmWtZtBOGWCkq+gKusTFUsHghjvHsPcQ2+EVfMcePBb/FKvtzSgH59C3iNOHE29l3ceSqccgxlxfStzbf+QkP7gxGVGZ8rLnCn3s8WzkGHZE4LtS0Zm3Y+hV5igrClk940YZP1hmilt2y7adPE4gCyQjb44JXgc3/NxlkZJcmeZTfAGxMXT8HG6Use/Kti114phsF7GDrqk1kPbB51Hr3xF1NAJUWP3csg3jgTS3E6jgD5XjPPG9BEDE2MwnBlUUMe3TC8TIWkK+AlwjlsDr5B9nqy2Fevv62+k5Adplw+fsQ8VzZREZF+MllWO3vtkD6srdx9h4vPD3dp5urFCFXNRaoD3SMDk27z3EVCQZ4bPL5PsVpB/ZBotLGkUZ0yi+5oC+u7ByP1ihMXMsRgvXbQpyOonEqDy84EZiIPWbyzGd0tEAXLz3mMh1x/IqZ1wxyDT/vkxhNCFqlBNlRW6GbMN2cng4A9Cigj9eNu9ptL1tdgFTxwndjoNRQMJ0NAc6WnsQ1UeIu8nMsa8/kLDtnVFLVmPQv2ZBUM4mxLrwC1mxOiQrWBW2XJ1OIheimSkLHfQOef1mIH3Z0cBuLBKGkRYGaXiZ6RX7po+ch0WFGjBef3e3uczl1mT5WGKdIG4x1+aRAtJHL+9K7Z6wzG0ygoamdiX2Fd0xBrWjTU72DzYbceqc+uHrbcLKDa5w0ENhyYK0+XEzG5fXHjFgmawY1D7xZQOJZO3jxStcv+xzoiTnNSrIxbxog/0Fez/WhMM9H6gV4eeDjMWEg79cJLugCBNwqmp3Yoe5EDU2TxQlLT53tye3Aji3FbocuDWjLI3Jc5VDxd7lrbzeIbFzSNpoFG8DSgjSiq41WJVeuzXxmdl7HM4zQpGRAAAAAQAFWC41MDkAAASsMIIEqDCCA5CgAwIBAgIJAJNurL4H8gHfMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTAeFw0wODAyMjkwMTMzNDZaFw0zNTA3MTcwMTMzNDZaMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgCggEBANaTGQTexgskse3HYuDZ2CU+Ps1s6x3i/waMqOi8qM1r03hupwqnbOYOuw+ZNVn/2T53qUPn6D1LZLjk/qLT5lbx4meoG7+yMLV4wgRDvkxyGLhG9SEVhvA4oU6Jwr44f46+z4/Kw9oe4zDJ6pPQp8PcSvNQIg1QCAcy4ICXF+5qBTNZ5qaU7Cyz8oSgpGbIepTYOzEJOmc3Li9kEsBubULxWBjf/gOBzAzURNps3cO4JFgZSAGzJWQTT7/emMkod0jb9WdqVA2BVMi7yge54kdVMxHEa5r3b97szI5p58ii0I54JiCUP5lyfTwE/nKZHZnfm644oLIXf6MdW2r+6R8CAQOjgfwwgfkwHQYDVR0OBBYEFEhZAFY9JyxGrhGGBaR0GawJyowRMIHJBgNVHSMEgcEwgb6AFEhZAFY9JyxGrhGGBaR0GawJyowRoYGapIGXMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbYIJAJNurL4H8gHfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHqvlozrUMRBBVEY0NqrrwFbinZaJ6cVosK0TyIUFf/azgMJWr+kLfcHCHJsIGnlw27drgQAvilFLAhLwn62oX6snb4YLCBOsVMR9FXYJLZW2+TcIkCRLXWG/oiVHQGo/rWuWkJgU134NDEFJCJGjDbiLCpe+ZTWHdcwauTJ9pUbo8EvHRkU3cYfGmLaLfgn9gP+pWA7LFQNvXwBnDa6sppCccEX31I828XzgXpJ4O+mDL1/dBd+ek8ZPUP0IgdyZm5MTYPhvVqGCHzzTy3sIeJFymwrsBbmg2OAUNLEMO6nwmocSdN2ClirfxqCzJOLSDE4QyS9BAH6EhY6UFcOaE21IJawTAEXnf52TqT7diFUlWRSnQ=='
+AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME = (
+    'debug_keystore_k47SVrA85+oMZAexHc62PkgvIgO8TJBYN00U82xSlxc_id_rsa'
+)
+
+
+class Options:
+    allow_disabled_algorithms = False
+    clean = False
+    delete_unknown = False
+    nosign = False
+    pretty = True
+    rename_apks = False
+    verbose = False
 
 
 class NightlyTest(unittest.TestCase):
+
+    basedir = Path(__file__).resolve().parent
+    path = os.environ['PATH']
+
+    def setUp(self):
+        common.config = None
+        nightly.config = None
+        logging.basicConfig(level=logging.WARNING)
+        self.basedir = Path(localmodule) / 'tests'
+        self.testroot = Path(localmodule) / '.testfiles'
+        self.testroot.mkdir(exist_ok=True)
+        os.chdir(self.basedir)
+        self.tempdir = tempfile.TemporaryDirectory(
+            str(time.time()), self._testMethodName + '_', self.testroot
+        )
+        self.testdir = Path(self.tempdir.name)
+        self.home = self.testdir / 'home'
+        self.home.mkdir()
+        self.dot_android = self.home / '.android'
+        nightly.KEYSTORE_FILE = str(self.dot_android / 'debug.keystore')
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    def _copy_test_debug_keystore(self):
+        self.dot_android.mkdir()
+        shutil.copy(
+            self.basedir / 'aosp_testkey_debug.keystore',
+            self.dot_android / 'debug.keystore',
+        )
+
+    def _copy_debug_apk(self):
+        outputdir = Path('app/build/output/apk/debug')
+        outputdir.mkdir(parents=True)
+        shutil.copy(self.basedir / 'urzip.apk', outputdir / 'urzip-debug.apk')
+
     def test_get_repo_base_url(self):
         for clone_url, repo_git_base, result in [
             (
@@ -37,6 +103,245 @@ class NightlyTest(unittest.TestCase):
             # gitlab.com often returns 403 Forbidden from their cloudflare restrictions
             self.assertTrue(r.status_code in (200, 403), 'should not be a redirect')
 
+    def test_get_keystore_secret_var(self):
+        self.assertEqual(
+            AOSP_TESTKEY_DEBUG_KEYSTORE,
+            nightly._get_keystore_secret_var(
+                self.basedir / 'aosp_testkey_debug.keystore'
+            ),
+        )
+
+    @patch.dict(os.environ, clear=True)
+    def test_ssh_key_from_debug_keystore(self):
+        os.environ['HOME'] = str(self.home)
+        os.environ['PATH'] = self.path
+        ssh_private_key_file = nightly._ssh_key_from_debug_keystore(
+            self.basedir / 'aosp_testkey_debug.keystore'
+        )
+        with open(ssh_private_key_file) as fp:
+            assert '-----BEGIN RSA PRIVATE KEY-----' in fp.read()
+        with open(ssh_private_key_file + '.pub') as fp:
+            assert fp.read(8) == 'ssh-rsa '
+
+    @patch.dict(os.environ, clear=True)
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    def test_main_empty_dot_android(self):
+        """Test that it exits with an error when ~/.android is empty"""
+        os.environ['HOME'] = str(self.home)
+        os.environ['PATH'] = self.path
+        with self.assertRaises(SystemExit) as cm:
+            nightly.main()
+        self.assertEqual(cm.exception.code, 1)
+
+    @patch.dict(os.environ, clear=True)
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    def test_main_empty_dot_ssh(self):
+        """Test that it does not create ~/.ssh if it does not exist
+
+        Careful!  If the test env is wrong, it can mess up the local
+        SSH setup.
+
+        """
+        dot_ssh = self.home / '.ssh'
+        self._copy_test_debug_keystore()
+        os.environ['HOME'] = str(self.home)
+        os.environ['PATH'] = self.path
+        assert not dot_ssh.exists()
+        nightly.main()
+        assert not dot_ssh.exists()
+
+    @patch.dict(os.environ, clear=True)
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    def test_main_on_user_machine(self):
+        """Test that `fdroid nightly` runs on the user's machine
+
+        Careful!  If the test env is wrong, it can mess up the local
+        SSH setup.
+
+        """
+        dot_ssh = self.home / '.ssh'
+        dot_ssh.mkdir()
+        self._copy_test_debug_keystore()
+        os.environ['HOME'] = str(self.home)
+        os.environ['PATH'] = self.path
+        nightly.main()
+        assert (dot_ssh / AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME).exists()
+        assert (dot_ssh / (AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME + '.pub')).exists()
+
+    @patch('fdroidserver.common.vcs_git.git', lambda args, e: common.PopenResult(1))
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    def test_private_or_non_existent_git_mirror(self):
+        """Test that this exits with an error when the git mirror repo won't work
+
+        Careful!  If the test environment is setup wrong, it can mess
+        up local files in ~/.ssh or ~/.android.
+
+        """
+        os.chdir(self.testdir)
+        with patch.dict(
+            os.environ,
+            {
+                'CI': 'true',
+                'CI_PROJECT_PATH': 'thisshouldneverexist/orthistoo',
+                'CI_PROJECT_URL': 'https://gitlab.com/thisshouldneverexist/orthistoo',
+                'DEBUG_KEYSTORE': DEBUG_KEYSTORE,
+                'GITLAB_USER_NAME': 'username',
+                'GITLAB_USER_EMAIL': 'username@example.com',
+                'HOME': str(self.testdir),
+                'PATH': os.getenv('PATH'),
+            },
+            clear=True,
+        ):
+            with self.assertRaises(exception.VCSException):
+                nightly.main()
+
+    def _put_fdroid_in_args(self, args):
+        """Find fdroid command that belongs to this source code tree"""
+        fdroid = os.path.join(localmodule, 'fdroid')
+        if not os.path.exists(fdroid):
+            fdroid = os.getenv('fdroid')
+        return [fdroid] + args[1:]
+
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    @patch('platform.node', lambda: 'example.com')
+    def test_github_actions(self):
+        """Careful! If the test env is bad, it'll mess up the local SSH setup
+
+        https://docs.github.com/en/actions/learn-github-actions/environment-variables
+
+        """
+
+        called = []
+        orig_check_call = subprocess.check_call
+        os.chdir(self.testdir)
+        os.makedirs('fdroid/git-mirror/fdroid/repo')  # fake this to avoid cloning
+        self._copy_test_debug_keystore()
+        self._copy_debug_apk()
+
+        def _subprocess_check_call(args, cwd=None, env=None):
+            if os.path.basename(args[0]) in ('keytool', 'openssl'):
+                orig_check_call(args, cwd=cwd, env=env)
+            elif args[:2] == ['fdroid', 'update']:
+                orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env)
+            else:
+                called.append(args[:2])
+                return
+
+        with patch.dict(
+            os.environ,
+            {
+                'CI': 'true',
+                'DEBUG_KEYSTORE': DEBUG_KEYSTORE,
+                'GITHUB_ACTIONS': 'true',
+                'GITHUB_ACTOR': 'username',
+                'GITHUB_REPOSITORY': 'f-droid/test',
+                'GITHUB_SERVER_URL': 'https://github.com',
+                'HOME': str(self.testdir),
+                'PATH': os.getenv('PATH'),
+                'fdroid': os.getenv('fdroid', ''),
+            },
+            clear=True,
+        ):
+            self.assertTrue(self.testroot == Path.home().parent)
+            with patch('subprocess.check_call', _subprocess_check_call):
+                nightly.main()
+        self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
+        self.assertFalse(os.path.exists('config.py'))
+        git_url = 'git@github.com:f-droid/test-nightly'
+        mirror_url = index.get_mirror_service_urls(git_url)[0]
+        expected = {
+            'archive_description': 'Old nightly builds that have been archived.',
+            'archive_name': 'f-droid/test-nightly archive',
+            'archive_older': 20,
+            'archive_url': mirror_url + '/archive',
+            'keydname': 'CN=Android Debug,O=Android,C=US',
+            'keypass': 'android',
+            'keystore': nightly.KEYSTORE_FILE,
+            'keystorepass': 'android',
+            'make_current_version_link': False,
+            'repo_description': 'Nightly builds from username@example.com',
+            'repo_keyalias': 'androiddebugkey',
+            'repo_name': 'f-droid/test-nightly',
+            'repo_url': mirror_url + '/repo',
+            'servergitmirrors': git_url,
+            'update_stats': True,
+        }
+        with open('config.yml') as fp:
+            config = yaml.safe_load(fp)
+            # .ssh is random tmpdir set in nightly.py, so test basename only
+            self.assertEqual(
+                os.path.basename(config['identity_file']),
+                DEBUG_KEYSTORE_KEY_FILE_NAME,
+            )
+            del config['identity_file']
+            self.assertEqual(expected, config)
+
+    @patch('sys.argv', ['fdroid nightly', '--verbose'])
+    def test_gitlab_ci(self):
+        """Careful!  If the test env is bad, it can mess up the local SSH setup"""
+        called = []
+        orig_check_call = subprocess.check_call
+        os.chdir(self.testdir)
+        os.makedirs('fdroid/git-mirror/fdroid/repo')  # fake this to avoid cloning
+        self._copy_test_debug_keystore()
+        self._copy_debug_apk()
+
+        def _subprocess_check_call(args, cwd=None, env=None):
+            if os.path.basename(args[0]) in ('keytool', 'openssl'):
+                orig_check_call(args, cwd=cwd, env=env)
+            elif args[:2] == ['fdroid', 'update']:
+                orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env)
+            else:
+                called.append(args[:2])
+                return
+
+        with patch.dict(
+            os.environ,
+            {
+                'CI': 'true',
+                'CI_PROJECT_PATH': 'fdroid/test',
+                'CI_PROJECT_URL': 'https://gitlab.com/fdroid/test',
+                'DEBUG_KEYSTORE': DEBUG_KEYSTORE,
+                'GITLAB_USER_NAME': 'username',
+                'GITLAB_USER_EMAIL': 'username@example.com',
+                'HOME': str(self.testdir),
+                'PATH': os.getenv('PATH'),
+                'fdroid': os.getenv('fdroid', ''),
+            },
+            clear=True,
+        ):
+            self.assertTrue(self.testroot == Path.home().parent)
+            with patch('subprocess.check_call', _subprocess_check_call):
+                nightly.main()
+        self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
+        self.assertFalse(os.path.exists('config.py'))
+        expected = {
+            'archive_description': 'Old nightly builds that have been archived.',
+            'archive_name': 'fdroid/test-nightly archive',
+            'archive_older': 20,
+            'archive_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/archive',
+            'keydname': 'CN=Android Debug,O=Android,C=US',
+            'keypass': 'android',
+            'keystore': nightly.KEYSTORE_FILE,
+            'keystorepass': 'android',
+            'make_current_version_link': False,
+            'repo_description': 'Nightly builds from username@example.com',
+            'repo_keyalias': 'androiddebugkey',
+            'repo_name': 'fdroid/test-nightly',
+            'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
+            'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly',
+            'update_stats': True,
+        }
+        with open('config.yml') as fp:
+            config = yaml.safe_load(fp)
+            # .ssh is random tmpdir set in nightly.py, so test basename only
+            self.assertEqual(
+                os.path.basename(config['identity_file']),
+                DEBUG_KEYSTORE_KEY_FILE_NAME,
+            )
+            del config['identity_file']
+            self.assertEqual(expected, config)
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))