diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94d49ed2..8c21edf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,11 +148,12 @@ ubuntu_jammy_pip: # back to bare machine to act as user's install machine - export ANDROID_HOME=/opt/android-sdk - $pip install sdkmanager - - sdkmanager 'build-tools;30.0.0' + - sdkmanager 'build-tools;33.0.0' - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* + - export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0 - fdroid=`which fdroid` ./tests/run-tests @@ -246,6 +247,7 @@ black: tests/lint.TestCase tests/metadata.TestCase tests/ndk-release-checksums.py + tests/nightly.TestCase tests/rewritemeta.TestCase tests/scanner.TestCase tests/signindex.TestCase diff --git a/MANIFEST.in b/MANIFEST.in index d1535466..87af42b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -42,6 +42,7 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po include locale/zh_Hant/LC_MESSAGES/fdroidserver.po include makebuildserver include README.md +include tests/aosp_testkey_debug.keystore include tests/apk.embedded_1.apk include tests/bad-unicode-*.apk include tests/build.TestCase @@ -623,6 +624,7 @@ include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml include tests/metadata/souch.smsbypass.yml include tests/metadata.TestCase include tests/minimal_targetsdk_30_unsigned.apk +include tests/nightly.TestCase include tests/Norway_bouvet_europe_2.obf.zip include tests/no_targetsdk_minsdk1_unsigned.apk include tests/no_targetsdk_minsdk30_unsigned.apk diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c882f891..5172f233 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2680,9 +2680,9 @@ def get_native_code(apkfile): class PopenResult: - def __init__(self): - self.returncode = None - self.output = None + def __init__(self, returncode=None, output=None): + self.returncode = returncode + self.output = output def SdkToolsPopen(commands, cwd=None, output=True): diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 1ca90ab4..13a8982b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -25,6 +25,7 @@ import re import subprocess import time import urllib +import yaml from argparse import ArgumentParser import logging import shutil @@ -376,7 +377,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): if repo_section == 'repo': git_mirror_path = 'git-mirror' dotgit = os.path.join(git_mirror_path, '.git') - git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section) + git_fdroiddir = os.path.join(git_mirror_path, 'fdroid') + git_repodir = os.path.join(git_fdroiddir, repo_section) if not os.path.isdir(git_repodir): os.makedirs(git_repodir) # github/gitlab use bare git repos, so only count the .git folder @@ -438,6 +440,18 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: progress = None + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + # push for every remote. This will overwrite the git history for remote in repo.remotes: if remote.name not in enabled_remotes: @@ -445,16 +459,22 @@ def update_servergitmirrors(servergitmirrors, repo_section): continue if remote.name == 'gitlab': logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') - with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file: - out_file.write("""pages: - script: - - mkdir .public - - cp -r * .public/ - - mv .public public - artifacts: - paths: - - public -""") + with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: + yaml.dump( + { + gitlab_ci_job_name: { + 'script': [ + 'mkdir .public', + 'cp -r * .public/', + 'mv .public public', + ], + 'artifacts': {'paths': ['public']}, + 'variables': {'GIT_DEPTH': 1}, + } + }, + fp, + default_flow_style=False, + ) repo.git.add(all=True) repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") diff --git a/fdroidserver/index.py b/fdroidserver/index.py index c788509d..59610d53 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1438,7 +1438,8 @@ def get_mirror_service_urls(url): segments.extend([branch, folder]) urls.append('/'.join(segments)) elif hostname == "gitlab.com": - if common.get_dir_size(folder) <= common.GITLAB_COM_PAGES_MAX_SIZE: + git_mirror_path = os.path.join('git-mirror', folder) + if common.get_dir_size(git_mirror_path) <= common.GITLAB_COM_PAGES_MAX_SIZE: # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] urls.append('/'.join(gitlab_pages)) @@ -1452,6 +1453,26 @@ def get_mirror_service_urls(url): # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder" gitlab_raw = segments + ['-', 'raw', branch, folder] urls.append('/'.join(gitlab_raw)) + # GitLab Artifacts "https://user.gitlab.io/-/repo/-/jobs/job_id/artifacts/public/folder" + job_id = os.getenv('CI_JOB_ID') + try: + int(job_id) + gitlab_artifacts = [ + "https:", + "", + user + ".gitlab.io", + '-', + repo, + '-', + 'jobs', + job_id, + 'artifacts', + 'public', + folder, + ] + urls.append('/'.join(gitlab_artifacts)) + except (TypeError, ValueError): + pass # no Job ID to use, ignore return urls diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 4d0d80ad..045f5496 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -25,6 +25,7 @@ import os import paramiko import platform import shutil +import ssl import subprocess import sys import tempfile @@ -34,7 +35,7 @@ from argparse import ArgumentParser from . import _ from . import common - +from .exception import VCSException # hard coded defaults for Android ~/.android/debug.keystore files # https://developers.google.com/android/guides/client-auth @@ -47,7 +48,16 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' -def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): +def _get_keystore_secret_var(keystore): + with open(keystore, 'rb') as fp: + return base64.standard_b64encode(fp.read()).decode('ascii') + + +def _ssh_key_from_debug_keystore(keystore=None): + if keystore is None: + # set this here so it can be overridden in the tests + # TODO convert this to a class to get rid of this nonsense + keystore = KEYSTORE_FILE tmp_dir = tempfile.mkdtemp(prefix='.') privkey = os.path.join(tmp_dir, '.privkey') key_pem = os.path.join(tmp_dir, '.key.pem') @@ -94,10 +104,17 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): ], env={'LC_ALL': 'C.UTF-8'}, ) + + # OpenSSL 3.0 changed the default output format from PKCS#1 to + # PKCS#8, which paramiko does not support. + # https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional + # https://github.com/paramiko/paramiko/issues/1015 + openssl_rsa_cmd = ['openssl', 'rsa'] + if ssl.OPENSSL_VERSION_INFO[0] >= 3: + openssl_rsa_cmd += ['-traditional'] subprocess.check_call( - [ - 'openssl', - 'rsa', + openssl_rsa_cmd + + [ '-in', key_pem, '-out', @@ -180,14 +197,16 @@ def main(): default=False, help=_("Don't use rsync checksums"), ) + archive_older_unset = -1 parser.add_argument( "--archive-older", type=int, - default=20, + default=archive_older_unset, help=_("Set maximum releases in repo before older ones are archived"), ) # TODO add --with-btlog options = parser.parse_args() + common.options = options # force a tighter umask since this writes private key material umask = os.umask(0o077) @@ -262,15 +281,17 @@ def main(): repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') - git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') - git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') - git_mirror_statsdir = os.path.join(git_mirror_path, 'fdroid', 'stats') + git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid') + git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') + git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) - try: - git.Repo.clone_from(clone_url, git_mirror_path) - except Exception: - pass + vcs = common.getvcs('git', clone_url, git_mirror_path) + p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)]) + if p.returncode != 0: + print('WARNING: only public git repos are supported!') + raise VCSException('git clone %s failed:' % clone_url, p.output) if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) @@ -316,28 +337,40 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, with open(ssh_config, 'a') as fp: fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file) - config = '' - config += "identity_file = '%s'\n" % ssh_private_key_file - config += "repo_name = '%s'\n" % repo_git_base - config += "repo_url = '%s'\n" % repo_url - config += "repo_description = 'Nightly builds from %s'\n" % git_user_email - config += "archive_name = '%s'\n" % (repo_git_base + ' archive') - config += "archive_url = '%s'\n" % (repo_base + '/archive') - config += ( - "archive_description = 'Old nightly builds that have been archived.'\n" - ) - config += "archive_older = %i\n" % options.archive_older - config += "servergitmirrors = '%s'\n" % servergitmirror - config += "keystore = '%s'\n" % KEYSTORE_FILE - config += "repo_keyalias = '%s'\n" % KEY_ALIAS - config += "keystorepass = '%s'\n" % PASSWORD - config += "keypass = '%s'\n" % PASSWORD - config += "keydname = '%s'\n" % DISTINGUISHED_NAME - config += "make_current_version_link = False\n" - config += "update_stats = True\n" - with open('config.py', 'w') as fp: - fp.write(config) - os.chmod('config.py', 0o600) + if options.archive_older == archive_older_unset: + fdroid_size = common.get_dir_size(git_mirror_fdroiddir) + max_size = common.GITLAB_COM_PAGES_MAX_SIZE + if fdroid_size < max_size: + options.archive_older = 20 + else: + options.archive_older = 3 + print( + 'WARNING: repo is %s over the GitLab Pages limit (%s)' + % (fdroid_size - max_size, max_size) + ) + print('Setting --archive-older to 3') + + config = { + 'identity_file': ssh_private_key_file, + 'repo_name': repo_git_base, + 'repo_url': repo_url, + 'repo_description': 'Nightly builds from %s' % git_user_email, + 'archive_name': repo_git_base + ' archive', + 'archive_url': repo_base + '/archive', + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_older': options.archive_older, + 'servergitmirrors': servergitmirror, + 'keystore': KEYSTORE_FILE, + 'repo_keyalias': KEY_ALIAS, + 'keystorepass': PASSWORD, + 'keypass': PASSWORD, + 'keydname': DISTINGUISHED_NAME, + 'make_current_version_link': False, + 'update_stats': True, + } + with open('config.yml', 'w') as fp: + yaml.dump(config, fp, default_flow_style=False) + os.chmod('config.yml', 0o600) config = common.read_config(options) common.assert_config_keystore(config) @@ -430,17 +463,16 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, + '\n -dname "CN=Android Debug,O=Android,C=US"') sys.exit(1) ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') - os.makedirs(os.path.dirname(ssh_dir), exist_ok=True) privkey = _ssh_key_from_debug_keystore(options.keystore) - ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) - shutil.move(privkey, ssh_private_key_file) - shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') + if os.path.exists(ssh_dir): + ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) + shutil.move(privkey, ssh_private_key_file) + shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') if shutil.rmtree.avoids_symlink_attacks: shutil.rmtree(os.path.dirname(privkey)) if options.show_secret_var: - with open(options.keystore, 'rb') as fp: - debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') + debug_keystore = _get_keystore_secret_var(options.keystore) print( _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format( path=options.keystore diff --git a/tests/aosp_testkey_debug.keystore b/tests/aosp_testkey_debug.keystore new file mode 100644 index 00000000..ecbdcb4d Binary files /dev/null and b/tests/aosp_testkey_debug.keystore differ diff --git a/tests/index.TestCase b/tests/index.TestCase index b04d81c0..1c8e9854 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -414,10 +414,17 @@ class IndexTest(unittest.TestCase): fdroidserver.index.get_mirror_service_urls(url), ) + @patch.dict(os.environ, clear=True) def test_gitlab_get_mirror_service_urls(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - os.mkdir('fdroid') - with Path('fdroid/placeholder').open('w') as fp: + git_mirror_path = Path('git-mirror/fdroid') + git_mirror_path.mkdir(parents=True) + ci_job_id = '12345678' + artifacts_url = ( + 'https://group.gitlab.io/-/project/-/jobs/%s/artifacts/public/fdroid' + % ci_job_id + ) + with (git_mirror_path / 'placeholder').open('w') as fp: fp.write(' ') for url in [ 'git@gitlab.com:group/project', @@ -426,20 +433,34 @@ class IndexTest(unittest.TestCase): 'https://gitlab.com/group/project.git', ]: with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 1000): + expected = [ + 'https://group.gitlab.io/project/fdroid', + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ] self.assertEqual( - [ - 'https://group.gitlab.io/project/fdroid', - 'https://gitlab.com/group/project/-/raw/master/fdroid', - ], + expected, fdroidserver.index.get_mirror_service_urls(url), ) + with patch.dict(os.environ, clear=True): + os.environ['CI_JOB_ID'] = ci_job_id + self.assertEqual( + expected + [artifacts_url], + fdroidserver.index.get_mirror_service_urls(url), + ) with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10): + expected = [ + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ] self.assertEqual( - [ - 'https://gitlab.com/group/project/-/raw/master/fdroid', - ], + expected, fdroidserver.index.get_mirror_service_urls(url), ) + with patch.dict(os.environ, clear=True): + os.environ['CI_JOB_ID'] = ci_job_id + self.assertEqual( + expected + [artifacts_url], + fdroidserver.index.get_mirror_service_urls(url), + ) def test_make_website(self): tmptestsdir = tempfile.mkdtemp( diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index cf3ccf8b..85a75abd 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -1,11 +1,20 @@ #!/usr/bin/env python3 import inspect +import logging import optparse import os import requests +import shutil +import subprocess import sys +import tempfile +import time import unittest +import yaml + +from pathlib import Path +from unittest.mock import patch localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') @@ -14,10 +23,67 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) -from fdroidserver import common, nightly +from fdroidserver import common, exception, index, nightly + + +DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABNYhAuskAAAK8MIICuDAOBgorBgEEASoCEQEBBQAEggKkqRnFlhidQmVff83bsAeewXPIsF0jiymzJnvrnUAQtCK0MV9uZonu37Mrj/qKLn56mf6QcvEoKvpCstZxzftgYYpAHWMVLM+hy2Z707QZEHlY7Ukppt8DItj+dXkeqGt7f8KzOb2AQwDbt9lm1fJb+MefLowTaubtvrLMcKIne43CbCu2D8HyN7RPWpEkVetA2Qgr5W4sa3tIUT80afqo9jzwJjKCspuxY9A1M8EIM3/kvyLo2B9r0cuWwRjYZXJ6gmTYI2ARNz0KQnCZUok14NDg+mZTb1B7AzRfb0lfjbA6grbzuAL+WaEpO8/LgGfuOh7QBZBT498TElOaFfQ9toQWA79wAmrQCm4OoFukpPIy2m/l6VjJSmlK5Q+CMOl/Au7OG1sUUCTvPaIr0XKnsiwDJ7a71n9garnPWHkvuWapSRCzCNgaUoGQjB+fTMJFFrwT8P1aLfM6onc3KNrDStoQZuYe5ngCLlNS56bENkVGvJBfdkboxtHZjqDXXON9jWGSOI527J3o2D5sjSVyx3T9XPrsL4TA/nBtdU+c/+M6aoASZR2VymzAKdMrGfj9kE5GXp8vv2vkJj9+OJ4Jm5yeczocc/Idtojjb1yg+sq1yY8kAQxgezpY1rpgi2jF3tSN01c23DNvAaSJLJX2ZuH8sD40ACc80Y1Qp1nUTdpwBZUeaeNruBwx4PHU8GnC71FwtiUpwNs0OoSl0pgDUJ3ODC5bs8B5QmW1wu1eg7I4mMSmCsNGW6VN3sFcu+WEqnmTxPoZombdFZKxsr2oq359Nn4bJ6Uc9PBz/sXsns7Zx1vND/oK/Jv5Y269UVAMeKX/eGpfnxzagW3tqGbOu12C2p9Azo5VxiU2fG/tmk2PjaG5hV/ywReco7I6C1p8OWM2fwAAAAEABVguNTA5AAAB6TCCAeUwggFOoAMCAQICBE89gTUwDQYJKoZIhvcNAQEFBQAwNzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0FuZHJvaWQxFjAUBgNVBAMTDUFuZHJvaWQgRGVidWcwHhcNMTIwMjE2MjIyMDM3WhcNNDIwMjA4MjIyMDM3WjA3MQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQW5kcm9pZDEWMBQGA1UEAxMNQW5kcm9pZCBEZWJ1ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA3AKU7S7JXhUjEwxWP1/LPHXieh61SaA/+xbpqsPA+yjGz1sAcGAyuG6bjNAVm56pq7nkjJzicX7Wi83nUBo58DEC/quxOLdy0C4PEOSAeTnTT1RJIwMDvOgiL1GFCErvQ7gCH6zuAID/JRFbN6nIkhDjs2DYnSBl7aJJf8wCLc0CAwEAATANBgkqhkiG9w0BAQUFAAOBgQAoq/TJffA0l+ZGf89xndmHdxrO6qi+TzSlByvLZ4eFfCovTh1iO+Edrd5V1yXGLxyyvdsadMAFZT8SaxMrP5xxhJ0nra0APWYLpA96M//auMhQBWPgqPntwgvEZuEH7f0kdItjBJ39yijbG8xfgwid6XqNUo0TDDkp/wNWKpJ9tJe+2PrGw1NAvrgSydoH2j8DI1Eq' +DEBUG_KEYSTORE_KEY_FILE_NAME = ( + 'debug_keystore_QW+xRCJDGHXyyFtgCW8QRajj+6uYmsLwGWpCfYqYQ5M_id_rsa' +) + +AOSP_TESTKEY_DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABejjuIU0AAAUBMIIE/TAOBgorBgEEASoCEQEBBQAEggTpvqhdBtq9D3jRUZGnhKLbFH1LMtCKqwGg25ETAEhvK1GVRNuWAHAUUedCnarjgeUy/zx9OsHuZq18KjUI115kWq/jxkf00fIg7wrOmXoyJf5Dbc7NGKjU64rRmppQEkJ417Lq4Uola9EBJ/WweEu6UTjTn5HcNl4mVloWKMBKNPkVfhZhAkXUyjiZ9rCVHMjLOVKG5vyTWZLwXpYR00Xz6VyzSunTyDza5oUOT/Fh7Gw74V7iNHANydkBHmH+UJ100p0vNPRFvt/3ABfMjkNbRXKNERnyN7NeBmCAOceuXjme/n0XLUidP9/NYk1yAmRJgUnauKD6UPSZYaUPuNSSdf4dD5fCQ7OVDq95e7vmqRDfrKUoWmtpndN7hbVl+OHVZXk2ngvXbvoS+F7ShsEfbq7+c37dnOcVrIlrY+wlOWX2jN42T+AkGt3AfA8zdIPdNgLGk64Op+aP4vGyLQqbuUEzOTNG9uExjGlamogPKFf93GAF83xv7AChYLR/9H+B1E955FL58bRuYOXVWJfLRsO/jyjXsilhBggo3VD1omRuOp98AkKP+P9JXCTswK7IZgvbMK3GB6QIzD20vlT0eK6JGLeWE7cXVn6oT26zvnqAjJ94PjS+YckMOExhqwCivPp1VaX6JzpQ1wr52OsGDUvconcjYrBEHBiY+UnMUk0Wj4mhZlJd1lpybZcWZ3vhTIlM0uMt4udl7t+zsgZ6BW97/pkGaa+QoxeTvgNlHGYyDYp8hveM3bCLXTHULw8mXUHxOJawq/J3E6vZ5/h2nzfmQmWtZtBOGWCkq+gKusTFUsHghjvHsPcQ2+EVfMcePBb/FKvtzSgH59C3iNOHE29l3ceSqccgxlxfStzbf+QkP7gxGVGZ8rLnCn3s8WzkGHZE4LtS0Zm3Y+hV5igrClk940YZP1hmilt2y7adPE4gCyQjb44JXgc3/NxlkZJcmeZTfAGxMXT8HG6Use/Kti114phsF7GDrqk1kPbB51Hr3xF1NAJUWP3csg3jgTS3E6jgD5XjPPG9BEDE2MwnBlUUMe3TC8TIWkK+AlwjlsDr5B9nqy2Fevv62+k5Adplw+fsQ8VzZREZF+MllWO3vtkD6srdx9h4vPD3dp5urFCFXNRaoD3SMDk27z3EVCQZ4bPL5PsVpB/ZBotLGkUZ0yi+5oC+u7ByP1ihMXMsRgvXbQpyOonEqDy84EZiIPWbyzGd0tEAXLz3mMh1x/IqZ1wxyDT/vkxhNCFqlBNlRW6GbMN2cng4A9Cigj9eNu9ptL1tdgFTxwndjoNRQMJ0NAc6WnsQ1UeIu8nMsa8/kLDtnVFLVmPQv2ZBUM4mxLrwC1mxOiQrWBW2XJ1OIheimSkLHfQOef1mIH3Z0cBuLBKGkRYGaXiZ6RX7po+ch0WFGjBef3e3uczl1mT5WGKdIG4x1+aRAtJHL+9K7Z6wzG0ygoamdiX2Fd0xBrWjTU72DzYbceqc+uHrbcLKDa5w0ENhyYK0+XEzG5fXHjFgmawY1D7xZQOJZO3jxStcv+xzoiTnNSrIxbxog/0Fez/WhMM9H6gV4eeDjMWEg79cJLugCBNwqmp3Yoe5EDU2TxQlLT53tye3Aji3FbocuDWjLI3Jc5VDxd7lrbzeIbFzSNpoFG8DSgjSiq41WJVeuzXxmdl7HM4zQpGRAAAAAQAFWC41MDkAAASsMIIEqDCCA5CgAwIBAgIJAJNurL4H8gHfMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTAeFw0wODAyMjkwMTMzNDZaFw0zNTA3MTcwMTMzNDZaMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgCggEBANaTGQTexgskse3HYuDZ2CU+Ps1s6x3i/waMqOi8qM1r03hupwqnbOYOuw+ZNVn/2T53qUPn6D1LZLjk/qLT5lbx4meoG7+yMLV4wgRDvkxyGLhG9SEVhvA4oU6Jwr44f46+z4/Kw9oe4zDJ6pPQp8PcSvNQIg1QCAcy4ICXF+5qBTNZ5qaU7Cyz8oSgpGbIepTYOzEJOmc3Li9kEsBubULxWBjf/gOBzAzURNps3cO4JFgZSAGzJWQTT7/emMkod0jb9WdqVA2BVMi7yge54kdVMxHEa5r3b97szI5p58ii0I54JiCUP5lyfTwE/nKZHZnfm644oLIXf6MdW2r+6R8CAQOjgfwwgfkwHQYDVR0OBBYEFEhZAFY9JyxGrhGGBaR0GawJyowRMIHJBgNVHSMEgcEwgb6AFEhZAFY9JyxGrhGGBaR0GawJyowRoYGapIGXMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbYIJAJNurL4H8gHfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHqvlozrUMRBBVEY0NqrrwFbinZaJ6cVosK0TyIUFf/azgMJWr+kLfcHCHJsIGnlw27drgQAvilFLAhLwn62oX6snb4YLCBOsVMR9FXYJLZW2+TcIkCRLXWG/oiVHQGo/rWuWkJgU134NDEFJCJGjDbiLCpe+ZTWHdcwauTJ9pUbo8EvHRkU3cYfGmLaLfgn9gP+pWA7LFQNvXwBnDa6sppCccEX31I828XzgXpJ4O+mDL1/dBd+ek8ZPUP0IgdyZm5MTYPhvVqGCHzzTy3sIeJFymwrsBbmg2OAUNLEMO6nwmocSdN2ClirfxqCzJOLSDE4QyS9BAH6EhY6UFcOaE21IJawTAEXnf52TqT7diFUlWRSnQ==' +AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME = ( + 'debug_keystore_k47SVrA85+oMZAexHc62PkgvIgO8TJBYN00U82xSlxc_id_rsa' +) + + +class Options: + allow_disabled_algorithms = False + clean = False + delete_unknown = False + nosign = False + pretty = True + rename_apks = False + verbose = False class NightlyTest(unittest.TestCase): + + basedir = Path(__file__).resolve().parent + path = os.environ['PATH'] + + def setUp(self): + common.config = None + nightly.config = None + logging.basicConfig(level=logging.WARNING) + self.basedir = Path(localmodule) / 'tests' + self.testroot = Path(localmodule) / '.testfiles' + self.testroot.mkdir(exist_ok=True) + os.chdir(self.basedir) + self.tempdir = tempfile.TemporaryDirectory( + str(time.time()), self._testMethodName + '_', self.testroot + ) + self.testdir = Path(self.tempdir.name) + self.home = self.testdir / 'home' + self.home.mkdir() + self.dot_android = self.home / '.android' + nightly.KEYSTORE_FILE = str(self.dot_android / 'debug.keystore') + + def tearDown(self): + self.tempdir.cleanup() + + def _copy_test_debug_keystore(self): + self.dot_android.mkdir() + shutil.copy( + self.basedir / 'aosp_testkey_debug.keystore', + self.dot_android / 'debug.keystore', + ) + + def _copy_debug_apk(self): + outputdir = Path('app/build/output/apk/debug') + outputdir.mkdir(parents=True) + shutil.copy(self.basedir / 'urzip.apk', outputdir / 'urzip-debug.apk') + def test_get_repo_base_url(self): for clone_url, repo_git_base, result in [ ( @@ -37,6 +103,245 @@ class NightlyTest(unittest.TestCase): # gitlab.com often returns 403 Forbidden from their cloudflare restrictions self.assertTrue(r.status_code in (200, 403), 'should not be a redirect') + def test_get_keystore_secret_var(self): + self.assertEqual( + AOSP_TESTKEY_DEBUG_KEYSTORE, + nightly._get_keystore_secret_var( + self.basedir / 'aosp_testkey_debug.keystore' + ), + ) + + @patch.dict(os.environ, clear=True) + def test_ssh_key_from_debug_keystore(self): + os.environ['HOME'] = str(self.home) + os.environ['PATH'] = self.path + ssh_private_key_file = nightly._ssh_key_from_debug_keystore( + self.basedir / 'aosp_testkey_debug.keystore' + ) + with open(ssh_private_key_file) as fp: + assert '-----BEGIN RSA PRIVATE KEY-----' in fp.read() + with open(ssh_private_key_file + '.pub') as fp: + assert fp.read(8) == 'ssh-rsa ' + + @patch.dict(os.environ, clear=True) + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_main_empty_dot_android(self): + """Test that it exits with an error when ~/.android is empty""" + os.environ['HOME'] = str(self.home) + os.environ['PATH'] = self.path + with self.assertRaises(SystemExit) as cm: + nightly.main() + self.assertEqual(cm.exception.code, 1) + + @patch.dict(os.environ, clear=True) + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_main_empty_dot_ssh(self): + """Test that it does not create ~/.ssh if it does not exist + + Careful! If the test env is wrong, it can mess up the local + SSH setup. + + """ + dot_ssh = self.home / '.ssh' + self._copy_test_debug_keystore() + os.environ['HOME'] = str(self.home) + os.environ['PATH'] = self.path + assert not dot_ssh.exists() + nightly.main() + assert not dot_ssh.exists() + + @patch.dict(os.environ, clear=True) + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_main_on_user_machine(self): + """Test that `fdroid nightly` runs on the user's machine + + Careful! If the test env is wrong, it can mess up the local + SSH setup. + + """ + dot_ssh = self.home / '.ssh' + dot_ssh.mkdir() + self._copy_test_debug_keystore() + os.environ['HOME'] = str(self.home) + os.environ['PATH'] = self.path + nightly.main() + assert (dot_ssh / AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME).exists() + assert (dot_ssh / (AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME + '.pub')).exists() + + @patch('fdroidserver.common.vcs_git.git', lambda args, e: common.PopenResult(1)) + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_private_or_non_existent_git_mirror(self): + """Test that this exits with an error when the git mirror repo won't work + + Careful! If the test environment is setup wrong, it can mess + up local files in ~/.ssh or ~/.android. + + """ + os.chdir(self.testdir) + with patch.dict( + os.environ, + { + 'CI': 'true', + 'CI_PROJECT_PATH': 'thisshouldneverexist/orthistoo', + 'CI_PROJECT_URL': 'https://gitlab.com/thisshouldneverexist/orthistoo', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITLAB_USER_NAME': 'username', + 'GITLAB_USER_EMAIL': 'username@example.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + }, + clear=True, + ): + with self.assertRaises(exception.VCSException): + nightly.main() + + def _put_fdroid_in_args(self, args): + """Find fdroid command that belongs to this source code tree""" + fdroid = os.path.join(localmodule, 'fdroid') + if not os.path.exists(fdroid): + fdroid = os.getenv('fdroid') + return [fdroid] + args[1:] + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + @patch('platform.node', lambda: 'example.com') + def test_github_actions(self): + """Careful! If the test env is bad, it'll mess up the local SSH setup + + https://docs.github.com/en/actions/learn-github-actions/environment-variables + + """ + + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITHUB_ACTIONS': 'true', + 'GITHUB_ACTOR': 'username', + 'GITHUB_REPOSITORY': 'f-droid/test', + 'GITHUB_SERVER_URL': 'https://github.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + git_url = 'git@github.com:f-droid/test-nightly' + mirror_url = index.get_mirror_service_urls(git_url)[0] + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'f-droid/test-nightly archive', + 'archive_older': 20, + 'archive_url': mirror_url + '/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'f-droid/test-nightly', + 'repo_url': mirror_url + '/repo', + 'servergitmirrors': git_url, + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_gitlab_ci(self): + """Careful! If the test env is bad, it can mess up the local SSH setup""" + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'CI_PROJECT_PATH': 'fdroid/test', + 'CI_PROJECT_URL': 'https://gitlab.com/fdroid/test', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITLAB_USER_NAME': 'username', + 'GITLAB_USER_EMAIL': 'username@example.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'fdroid/test-nightly archive', + 'archive_older': 20, + 'archive_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'fdroid/test-nightly', + 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', + 'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly', + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))