From 553c9aa7e70d13568b45552b1626fc89e475fcd8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Nov 2022 23:25:21 +0100 Subject: [PATCH 01/13] include tests/nightly.TestCase in dist tarball --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index d1535466..048c139e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -623,6 +623,7 @@ include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml include tests/metadata/souch.smsbypass.yml include tests/metadata.TestCase include tests/minimal_targetsdk_30_unsigned.apk +include tests/nightly.TestCase include tests/Norway_bouvet_europe_2.obf.zip include tests/no_targetsdk_minsdk1_unsigned.apk include tests/no_targetsdk_minsdk30_unsigned.apk From bf945a3062e7e4ab78f389977e03d72ae44be228 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Sep 2022 22:51:31 +0200 Subject: [PATCH 02/13] nightly: only write SSH key files if ~/.ssh exists on dev's machine --- fdroidserver/nightly.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 4d0d80ad..bfeca16f 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -430,11 +430,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, + '\n -dname "CN=Android Debug,O=Android,C=US"') sys.exit(1) ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') - os.makedirs(os.path.dirname(ssh_dir), exist_ok=True) privkey = _ssh_key_from_debug_keystore(options.keystore) - ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) - shutil.move(privkey, ssh_private_key_file) - shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') + if os.path.exists(ssh_dir): + ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) + shutil.move(privkey, ssh_private_key_file) + shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') if shutil.rmtree.avoids_symlink_attacks: shutil.rmtree(os.path.dirname(privkey)) From 1c5506ae054b1590cca28e5d6c0c9ad923c3ae22 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Sep 2022 23:48:12 +0200 Subject: [PATCH 03/13] nightly: support OpenSSL 3.0 with Paramiko OpenSSL 3.0 changed the default output format from PKCS#1 to PKCS#8, which paramiko does not support. https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional https://github.com/paramiko/paramiko/issues/1015 --- .gitlab-ci.yml | 1 + MANIFEST.in | 1 + fdroidserver/nightly.py | 20 ++++-- tests/aosp_testkey_debug.keystore | Bin 0 -> 2557 bytes tests/nightly.TestCase | 108 ++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 tests/aosp_testkey_debug.keystore diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94d49ed2..de48b3c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -246,6 +246,7 @@ black: tests/lint.TestCase tests/metadata.TestCase tests/ndk-release-checksums.py + tests/nightly.TestCase tests/rewritemeta.TestCase tests/scanner.TestCase tests/signindex.TestCase diff --git a/MANIFEST.in b/MANIFEST.in index 048c139e..87af42b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -42,6 +42,7 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po include locale/zh_Hant/LC_MESSAGES/fdroidserver.po include makebuildserver include README.md +include tests/aosp_testkey_debug.keystore include tests/apk.embedded_1.apk include tests/bad-unicode-*.apk include tests/build.TestCase diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index bfeca16f..14db7fbd 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -25,6 +25,7 @@ import os import paramiko import platform import shutil +import ssl import subprocess import sys import tempfile @@ -47,7 +48,11 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' -def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): +def _ssh_key_from_debug_keystore(keystore=None): + if keystore is None: + # set this here so it can be overridden in the tests + # TODO convert this to a class to get rid of this nonsense + keystore = KEYSTORE_FILE tmp_dir = tempfile.mkdtemp(prefix='.') privkey = os.path.join(tmp_dir, '.privkey') key_pem = os.path.join(tmp_dir, '.key.pem') @@ -94,10 +99,17 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): ], env={'LC_ALL': 'C.UTF-8'}, ) + + # OpenSSL 3.0 changed the default output format from PKCS#1 to + # PKCS#8, which paramiko does not support. + # https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional + # https://github.com/paramiko/paramiko/issues/1015 + openssl_rsa_cmd = ['openssl', 'rsa'] + if ssl.OPENSSL_VERSION_INFO[0] >= 3: + openssl_rsa_cmd += ['-traditional'] subprocess.check_call( - [ - 'openssl', - 'rsa', + openssl_rsa_cmd + + [ '-in', key_pem, '-out', diff --git a/tests/aosp_testkey_debug.keystore b/tests/aosp_testkey_debug.keystore new file mode 100644 index 0000000000000000000000000000000000000000..ecbdcb4dc59dfbacd33f24d6f1690900c6ee16b0 GIT binary patch literal 2557 zcmd6o`8(A88pr25Gsf1C21Ajd5n1XxcCr+aeG3nh9D^a-sIg`rYKlB0Tcf6IMGx7F z!r0f;v6X$-NJu1GH6A^ub57TF&L8mna9{Uzzwgfv_vdrH@7IU5%vy#(Ake)4{t8|K z$&Kvi>E=ds3H0zLUfI(_)jo=wK_F}Z7z6(f@*ucm5da*Jf${+W8w4H$UtknmK#bJ! zhV)t`=U>ek<&Q8?=!q``vPP3p5JNv%tF$wt>5xnOR`a^_KV~WK6m~TuG#Wp;KM9vn z<`{Hx!n+xc$i~ov&a2^MYfA z@j`!x<_Pl|kHsos$>-#>(~Qz3Walt&n83262lUZTD6Q)=CO0AI?nJ%w`sCmFgx6Fn zoDj~@>g~l&>i1a@Xn8}r&g3f>Gcwf8h+;HSowZ}RP9r?ztsb}?$P(qvXws`l<0-ik zR>OR}f3oPIG~U~j-XC`DDRFE14e^S~eeOZh&VenbK5C;FshH&-Ieu?mpeWAUbxBg4 zxREfSVljn1INlJ>s~8YFt^#dKDdx#%Ce5st%X8g*ep*po(O_d5m*9!dyumB|+xIbN zw$I^yOOR~m%Bb}y%7Dd{IqU7!N|N)tH})AX4VvqdqzB875fB{GyVAXP9zqZ1y4Xe%)Yo%_^G z35AfH+MPh#hdWXlmV;c|LyHKc$yiSC5tq_4B0_hzjk>MP1e4;j-K{UY#glzA1U*s+`HZe(1zcIxs?Po?p^y^(XQPqh}@4T&ozvun%41?L#58 zpEiT;lZq^_IS$~nwE94Gl@+ZP8?5kj#ml*`0(oM?h?^z{^@aPT8O&%#O(|K&J{#vR zXTUw^%SG0RYbnsIoicF2e7XA)NAK%}IM#07=?H9JlksrGbtvsJObJTjUa2UeK8fnn z97GOLgZ1RZ=s2mYcvjZ=1_9^VImZ)Ytxh!sD6wnc!}h;9d!weayR1a#R_QX`%EZpK zr{4Xv0hI<=GatEl%f(XwL+JeB> zM@jdt>t8zvIz z%r|&d4qg6mZ)bYZx2X$Re5pr|&>2&?eOd8P#^7O`Gp$HyKzo%4i*s9^ZIyMbU-Hkv z&MV7wwANmX{mvGq^Y&`9mRNzn^n7eW>($tLM{G?N2kO!vUcoNcpYK;zvEY}K*AA|d zs)DLj3A_-gR?e0Cz0*HUuXW1@iqUI8a=A1Rqa#KQhUafD;7ye~9cTq?unJDS+Tf!q|BdgK_)! zh`XnAq1aFVi_HI4xc^6i{a29yG5?u?6A(oGr246!`M9f}FL)S@1l2$V1$7XosHmiZ zMPAXp55gn%&s2Q&r%Aa7HI;S+7#*s|qzm#N_)Np0;{ zK8vC=yNHB>_q7GDy!%5)`CR!vOr9EEnzF<0uy*icz4`ZACT`Vptek$P-Rg`-!J+zc z@M%aBT#sQ&7OFP*A}&Dvq?ThTk92O3Dex~uj=lPnmed*8sB5ufqq zYOKAmAy6UdhO(%ixYH>eY&iPG!^;LqvFWJkVt+n!)>e_P#XED;Z(^xC(R02drzbJw zC?-{hMvl;gv&b}2+T`70wXAY6oBPYzJZi&P;}2YqC9XxIKQC-#7;{} z&Y+l&%9a-(T*TRxzz0$~fAo*QDUBe75&eJ9WK%NpC>eiKC&c`+ar=H;1qEjVVF0lE zt3CGiZ_ge8B(x+wVbQYXG@F%B&q!eja6Uc=FO@Hl(^P38!7s2o@*2j8ug{a;Wal9J zU_9S7lg5kT5XLcmIS!Mi$YVdz>oP&ULr>QL0tBQQ3qqXQ*IjK)by}*aA4 Date: Fri, 11 Nov 2022 10:14:39 +0100 Subject: [PATCH 04/13] nightly: convert to config.yml --- fdroidserver/nightly.py | 43 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 14db7fbd..2ce83557 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -328,28 +328,27 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, with open(ssh_config, 'a') as fp: fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file) - config = '' - config += "identity_file = '%s'\n" % ssh_private_key_file - config += "repo_name = '%s'\n" % repo_git_base - config += "repo_url = '%s'\n" % repo_url - config += "repo_description = 'Nightly builds from %s'\n" % git_user_email - config += "archive_name = '%s'\n" % (repo_git_base + ' archive') - config += "archive_url = '%s'\n" % (repo_base + '/archive') - config += ( - "archive_description = 'Old nightly builds that have been archived.'\n" - ) - config += "archive_older = %i\n" % options.archive_older - config += "servergitmirrors = '%s'\n" % servergitmirror - config += "keystore = '%s'\n" % KEYSTORE_FILE - config += "repo_keyalias = '%s'\n" % KEY_ALIAS - config += "keystorepass = '%s'\n" % PASSWORD - config += "keypass = '%s'\n" % PASSWORD - config += "keydname = '%s'\n" % DISTINGUISHED_NAME - config += "make_current_version_link = False\n" - config += "update_stats = True\n" - with open('config.py', 'w') as fp: - fp.write(config) - os.chmod('config.py', 0o600) + config = { + 'identity_file': ssh_private_key_file, + 'repo_name': repo_git_base, + 'repo_url': repo_url, + 'repo_description': 'Nightly builds from %s' % git_user_email, + 'archive_name': repo_git_base + ' archive', + 'archive_url': repo_base + '/archive', + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_older': options.archive_older, + 'servergitmirrors': servergitmirror, + 'keystore': KEYSTORE_FILE, + 'repo_keyalias': KEY_ALIAS, + 'keystorepass': PASSWORD, + 'keypass': PASSWORD, + 'keydname': DISTINGUISHED_NAME, + 'make_current_version_link': False, + 'update_stats': True, + } + with open('config.yml', 'w') as fp: + yaml.dump(config, fp, default_flow_style=False) + os.chmod('config.yml', 0o600) config = common.read_config(options) common.assert_config_keystore(config) From 15bd7057f0a2037ef434325c952a3dcc66ffe0f7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Nov 2022 23:25:21 +0100 Subject: [PATCH 05/13] nightly: add tests --- .gitlab-ci.yml | 1 + fdroidserver/nightly.py | 8 +- tests/nightly.TestCase | 172 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 178 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de48b3c8..20a82096 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,6 +153,7 @@ ubuntu_jammy_pip: - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* + - export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0 - fdroid=`which fdroid` ./tests/run-tests diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 2ce83557..d1f69e40 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -48,6 +48,11 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' +def _get_keystore_secret_var(keystore): + with open(keystore, 'rb') as fp: + return base64.standard_b64encode(fp.read()).decode('ascii') + + def _ssh_key_from_debug_keystore(keystore=None): if keystore is None: # set this here so it can be overridden in the tests @@ -450,8 +455,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.rmtree(os.path.dirname(privkey)) if options.show_secret_var: - with open(options.keystore, 'rb') as fp: - debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') + debug_keystore = _get_keystore_secret_var(options.keystore) print( _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format( path=options.keystore diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index c5fbf0cd..ab854faf 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -6,10 +6,12 @@ import optparse import os import requests import shutil +import subprocess import sys import tempfile import time import unittest +import yaml from pathlib import Path from unittest.mock import patch @@ -21,9 +23,15 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) -from fdroidserver import common, nightly +from fdroidserver import common, index, nightly +DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABNYhAuskAAAK8MIICuDAOBgorBgEEASoCEQEBBQAEggKkqRnFlhidQmVff83bsAeewXPIsF0jiymzJnvrnUAQtCK0MV9uZonu37Mrj/qKLn56mf6QcvEoKvpCstZxzftgYYpAHWMVLM+hy2Z707QZEHlY7Ukppt8DItj+dXkeqGt7f8KzOb2AQwDbt9lm1fJb+MefLowTaubtvrLMcKIne43CbCu2D8HyN7RPWpEkVetA2Qgr5W4sa3tIUT80afqo9jzwJjKCspuxY9A1M8EIM3/kvyLo2B9r0cuWwRjYZXJ6gmTYI2ARNz0KQnCZUok14NDg+mZTb1B7AzRfb0lfjbA6grbzuAL+WaEpO8/LgGfuOh7QBZBT498TElOaFfQ9toQWA79wAmrQCm4OoFukpPIy2m/l6VjJSmlK5Q+CMOl/Au7OG1sUUCTvPaIr0XKnsiwDJ7a71n9garnPWHkvuWapSRCzCNgaUoGQjB+fTMJFFrwT8P1aLfM6onc3KNrDStoQZuYe5ngCLlNS56bENkVGvJBfdkboxtHZjqDXXON9jWGSOI527J3o2D5sjSVyx3T9XPrsL4TA/nBtdU+c/+M6aoASZR2VymzAKdMrGfj9kE5GXp8vv2vkJj9+OJ4Jm5yeczocc/Idtojjb1yg+sq1yY8kAQxgezpY1rpgi2jF3tSN01c23DNvAaSJLJX2ZuH8sD40ACc80Y1Qp1nUTdpwBZUeaeNruBwx4PHU8GnC71FwtiUpwNs0OoSl0pgDUJ3ODC5bs8B5QmW1wu1eg7I4mMSmCsNGW6VN3sFcu+WEqnmTxPoZombdFZKxsr2oq359Nn4bJ6Uc9PBz/sXsns7Zx1vND/oK/Jv5Y269UVAMeKX/eGpfnxzagW3tqGbOu12C2p9Azo5VxiU2fG/tmk2PjaG5hV/ywReco7I6C1p8OWM2fwAAAAEABVguNTA5AAAB6TCCAeUwggFOoAMCAQICBE89gTUwDQYJKoZIhvcNAQEFBQAwNzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0FuZHJvaWQxFjAUBgNVBAMTDUFuZHJvaWQgRGVidWcwHhcNMTIwMjE2MjIyMDM3WhcNNDIwMjA4MjIyMDM3WjA3MQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQW5kcm9pZDEWMBQGA1UEAxMNQW5kcm9pZCBEZWJ1ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA3AKU7S7JXhUjEwxWP1/LPHXieh61SaA/+xbpqsPA+yjGz1sAcGAyuG6bjNAVm56pq7nkjJzicX7Wi83nUBo58DEC/quxOLdy0C4PEOSAeTnTT1RJIwMDvOgiL1GFCErvQ7gCH6zuAID/JRFbN6nIkhDjs2DYnSBl7aJJf8wCLc0CAwEAATANBgkqhkiG9w0BAQUFAAOBgQAoq/TJffA0l+ZGf89xndmHdxrO6qi+TzSlByvLZ4eFfCovTh1iO+Edrd5V1yXGLxyyvdsadMAFZT8SaxMrP5xxhJ0nra0APWYLpA96M//auMhQBWPgqPntwgvEZuEH7f0kdItjBJ39yijbG8xfgwid6XqNUo0TDDkp/wNWKpJ9tJe+2PrGw1NAvrgSydoH2j8DI1Eq' +DEBUG_KEYSTORE_KEY_FILE_NAME = ( + 'debug_keystore_QW+xRCJDGHXyyFtgCW8QRajj+6uYmsLwGWpCfYqYQ5M_id_rsa' +) + +AOSP_TESTKEY_DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABejjuIU0AAAUBMIIE/TAOBgorBgEEASoCEQEBBQAEggTpvqhdBtq9D3jRUZGnhKLbFH1LMtCKqwGg25ETAEhvK1GVRNuWAHAUUedCnarjgeUy/zx9OsHuZq18KjUI115kWq/jxkf00fIg7wrOmXoyJf5Dbc7NGKjU64rRmppQEkJ417Lq4Uola9EBJ/WweEu6UTjTn5HcNl4mVloWKMBKNPkVfhZhAkXUyjiZ9rCVHMjLOVKG5vyTWZLwXpYR00Xz6VyzSunTyDza5oUOT/Fh7Gw74V7iNHANydkBHmH+UJ100p0vNPRFvt/3ABfMjkNbRXKNERnyN7NeBmCAOceuXjme/n0XLUidP9/NYk1yAmRJgUnauKD6UPSZYaUPuNSSdf4dD5fCQ7OVDq95e7vmqRDfrKUoWmtpndN7hbVl+OHVZXk2ngvXbvoS+F7ShsEfbq7+c37dnOcVrIlrY+wlOWX2jN42T+AkGt3AfA8zdIPdNgLGk64Op+aP4vGyLQqbuUEzOTNG9uExjGlamogPKFf93GAF83xv7AChYLR/9H+B1E955FL58bRuYOXVWJfLRsO/jyjXsilhBggo3VD1omRuOp98AkKP+P9JXCTswK7IZgvbMK3GB6QIzD20vlT0eK6JGLeWE7cXVn6oT26zvnqAjJ94PjS+YckMOExhqwCivPp1VaX6JzpQ1wr52OsGDUvconcjYrBEHBiY+UnMUk0Wj4mhZlJd1lpybZcWZ3vhTIlM0uMt4udl7t+zsgZ6BW97/pkGaa+QoxeTvgNlHGYyDYp8hveM3bCLXTHULw8mXUHxOJawq/J3E6vZ5/h2nzfmQmWtZtBOGWCkq+gKusTFUsHghjvHsPcQ2+EVfMcePBb/FKvtzSgH59C3iNOHE29l3ceSqccgxlxfStzbf+QkP7gxGVGZ8rLnCn3s8WzkGHZE4LtS0Zm3Y+hV5igrClk940YZP1hmilt2y7adPE4gCyQjb44JXgc3/NxlkZJcmeZTfAGxMXT8HG6Use/Kti114phsF7GDrqk1kPbB51Hr3xF1NAJUWP3csg3jgTS3E6jgD5XjPPG9BEDE2MwnBlUUMe3TC8TIWkK+AlwjlsDr5B9nqy2Fevv62+k5Adplw+fsQ8VzZREZF+MllWO3vtkD6srdx9h4vPD3dp5urFCFXNRaoD3SMDk27z3EVCQZ4bPL5PsVpB/ZBotLGkUZ0yi+5oC+u7ByP1ihMXMsRgvXbQpyOonEqDy84EZiIPWbyzGd0tEAXLz3mMh1x/IqZ1wxyDT/vkxhNCFqlBNlRW6GbMN2cng4A9Cigj9eNu9ptL1tdgFTxwndjoNRQMJ0NAc6WnsQ1UeIu8nMsa8/kLDtnVFLVmPQv2ZBUM4mxLrwC1mxOiQrWBW2XJ1OIheimSkLHfQOef1mIH3Z0cBuLBKGkRYGaXiZ6RX7po+ch0WFGjBef3e3uczl1mT5WGKdIG4x1+aRAtJHL+9K7Z6wzG0ygoamdiX2Fd0xBrWjTU72DzYbceqc+uHrbcLKDa5w0ENhyYK0+XEzG5fXHjFgmawY1D7xZQOJZO3jxStcv+xzoiTnNSrIxbxog/0Fez/WhMM9H6gV4eeDjMWEg79cJLugCBNwqmp3Yoe5EDU2TxQlLT53tye3Aji3FbocuDWjLI3Jc5VDxd7lrbzeIbFzSNpoFG8DSgjSiq41WJVeuzXxmdl7HM4zQpGRAAAAAQAFWC41MDkAAASsMIIEqDCCA5CgAwIBAgIJAJNurL4H8gHfMA0GCSqGSIb3DQEBBQUAMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTAeFw0wODAyMjkwMTMzNDZaFw0zNTA3MTcwMTMzNDZaMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgCggEBANaTGQTexgskse3HYuDZ2CU+Ps1s6x3i/waMqOi8qM1r03hupwqnbOYOuw+ZNVn/2T53qUPn6D1LZLjk/qLT5lbx4meoG7+yMLV4wgRDvkxyGLhG9SEVhvA4oU6Jwr44f46+z4/Kw9oe4zDJ6pPQp8PcSvNQIg1QCAcy4ICXF+5qBTNZ5qaU7Cyz8oSgpGbIepTYOzEJOmc3Li9kEsBubULxWBjf/gOBzAzURNps3cO4JFgZSAGzJWQTT7/emMkod0jb9WdqVA2BVMi7yge54kdVMxHEa5r3b97szI5p58ii0I54JiCUP5lyfTwE/nKZHZnfm644oLIXf6MdW2r+6R8CAQOjgfwwgfkwHQYDVR0OBBYEFEhZAFY9JyxGrhGGBaR0GawJyowRMIHJBgNVHSMEgcEwgb6AFEhZAFY9JyxGrhGGBaR0GawJyowRoYGapIGXMIGUMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHQW5kcm9pZDEQMA4GA1UECxMHQW5kcm9pZDEQMA4GA1UEAxMHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbYIJAJNurL4H8gHfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHqvlozrUMRBBVEY0NqrrwFbinZaJ6cVosK0TyIUFf/azgMJWr+kLfcHCHJsIGnlw27drgQAvilFLAhLwn62oX6snb4YLCBOsVMR9FXYJLZW2+TcIkCRLXWG/oiVHQGo/rWuWkJgU134NDEFJCJGjDbiLCpe+ZTWHdcwauTJ9pUbo8EvHRkU3cYfGmLaLfgn9gP+pWA7LFQNvXwBnDa6sppCccEX31I828XzgXpJ4O+mDL1/dBd+ek8ZPUP0IgdyZm5MTYPhvVqGCHzzTy3sIeJFymwrsBbmg2OAUNLEMO6nwmocSdN2ClirfxqCzJOLSDE4QyS9BAH6EhY6UFcOaE21IJawTAEXnf52TqT7diFUlWRSnQ==' AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME = ( 'debug_keystore_k47SVrA85+oMZAexHc62PkgvIgO8TJBYN00U82xSlxc_id_rsa' ) @@ -45,6 +53,8 @@ class NightlyTest(unittest.TestCase): path = os.environ['PATH'] def setUp(self): + common.config = None + nightly.config = None logging.basicConfig(level=logging.WARNING) self.basedir = Path(localmodule) / 'tests' self.testroot = Path(localmodule) / '.testfiles' @@ -69,6 +79,11 @@ class NightlyTest(unittest.TestCase): self.dot_android / 'debug.keystore', ) + def _copy_debug_apk(self): + outputdir = Path('app/build/output/apk/debug') + outputdir.mkdir(parents=True) + shutil.copy(self.basedir / 'urzip.apk', outputdir / 'urzip-debug.apk') + def test_get_repo_base_url(self): for clone_url, repo_git_base, result in [ ( @@ -88,6 +103,14 @@ class NightlyTest(unittest.TestCase): # gitlab.com often returns 403 Forbidden from their cloudflare restrictions self.assertTrue(r.status_code in (200, 403), 'should not be a redirect') + def test_get_keystore_secret_var(self): + self.assertEqual( + AOSP_TESTKEY_DEBUG_KEYSTORE, + nightly._get_keystore_secret_var( + self.basedir / 'aosp_testkey_debug.keystore' + ), + ) + @patch.dict(os.environ, clear=True) def test_ssh_key_from_debug_keystore(self): os.environ['HOME'] = str(self.home) @@ -145,6 +168,153 @@ class NightlyTest(unittest.TestCase): assert (dot_ssh / AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME).exists() assert (dot_ssh / (AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME + '.pub')).exists() + def _put_fdroid_in_args(self, args): + """Find fdroid command that belongs to this source code tree""" + fdroid = os.path.join(localmodule, 'fdroid') + if not os.path.exists(fdroid): + fdroid = os.getenv('fdroid') + return [fdroid] + args[1:] + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + @patch('platform.node', lambda: 'example.com') + def test_github_actions(self): + """Careful! If the test env is bad, it'll mess up the local SSH setup + + https://docs.github.com/en/actions/learn-github-actions/environment-variables + + """ + + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITHUB_ACTIONS': 'true', + 'GITHUB_ACTOR': 'username', + 'GITHUB_REPOSITORY': 'f-droid/test', + 'GITHUB_SERVER_URL': 'https://github.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + git_url = 'git@github.com:f-droid/test-nightly' + mirror_url = index.get_mirror_service_urls(git_url)[0] + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'f-droid/test-nightly archive', + 'archive_older': 20, + 'archive_url': mirror_url + '/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'f-droid/test-nightly', + 'repo_url': mirror_url + '/repo', + 'servergitmirrors': git_url, + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_gitlab_ci(self): + """Careful! If the test env is bad, it can mess up the local SSH setup""" + called = [] + orig_check_call = subprocess.check_call + os.chdir(self.testdir) + os.makedirs('fdroid/git-mirror/fdroid/repo') # fake this to avoid cloning + self._copy_test_debug_keystore() + self._copy_debug_apk() + + def _subprocess_check_call(args, cwd=None, env=None): + if os.path.basename(args[0]) in ('keytool', 'openssl'): + orig_check_call(args, cwd=cwd, env=env) + elif args[:2] == ['fdroid', 'update']: + orig_check_call(self._put_fdroid_in_args(args), cwd=cwd, env=env) + else: + called.append(args[:2]) + return + + with patch.dict( + os.environ, + { + 'CI': 'true', + 'CI_PROJECT_PATH': 'fdroid/test', + 'CI_PROJECT_URL': 'https://gitlab.com/fdroid/test', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITLAB_USER_NAME': 'username', + 'GITLAB_USER_EMAIL': 'username@example.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + 'fdroid': os.getenv('fdroid', ''), + }, + clear=True, + ): + self.assertTrue(self.testroot == Path.home().parent) + with patch('subprocess.check_call', _subprocess_check_call): + nightly.main() + self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) + self.assertFalse(os.path.exists('config.py')) + expected = { + 'archive_description': 'Old nightly builds that have been archived.', + 'archive_name': 'fdroid/test-nightly archive', + 'archive_older': 20, + 'archive_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/archive', + 'keydname': 'CN=Android Debug,O=Android,C=US', + 'keypass': 'android', + 'keystore': nightly.KEYSTORE_FILE, + 'keystorepass': 'android', + 'make_current_version_link': False, + 'repo_description': 'Nightly builds from username@example.com', + 'repo_keyalias': 'androiddebugkey', + 'repo_name': 'fdroid/test-nightly', + 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', + 'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly', + 'update_stats': True, + } + with open('config.yml') as fp: + config = yaml.safe_load(fp) + # .ssh is random tmpdir set in nightly.py, so test basename only + self.assertEqual( + os.path.basename(config['identity_file']), + DEBUG_KEYSTORE_KEY_FILE_NAME, + ) + del config['identity_file'] + self.assertEqual(expected, config) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From c2567d71d16deb2c78278e2b629830fcc4864daf Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Nov 2022 23:13:13 +0100 Subject: [PATCH 06/13] nightly: fail if *-nightly git repo is not publicly available --- fdroidserver/common.py | 6 +++--- fdroidserver/nightly.py | 12 +++++++----- tests/nightly.TestCase | 29 ++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c882f891..5172f233 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2680,9 +2680,9 @@ def get_native_code(apkfile): class PopenResult: - def __init__(self): - self.returncode = None - self.output = None + def __init__(self, returncode=None, output=None): + self.returncode = returncode + self.output = output def SdkToolsPopen(commands, cwd=None, output=True): diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index d1f69e40..8e11fa0b 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -35,7 +35,7 @@ from argparse import ArgumentParser from . import _ from . import common - +from .exception import VCSException # hard coded defaults for Android ~/.android/debug.keystore files # https://developers.google.com/android/guides/client-auth @@ -205,6 +205,7 @@ def main(): ) # TODO add --with-btlog options = parser.parse_args() + common.options = options # force a tighter umask since this writes private key material umask = os.umask(0o077) @@ -284,10 +285,11 @@ def main(): git_mirror_statsdir = os.path.join(git_mirror_path, 'fdroid', 'stats') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) - try: - git.Repo.clone_from(clone_url, git_mirror_path) - except Exception: - pass + vcs = common.getvcs('git', clone_url, git_mirror_path) + p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)]) + if p.returncode != 0: + print('WARNING: only public git repos are supported!') + raise VCSException('git clone %s failed:' % clone_url, p.output) if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index ab854faf..85a75abd 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -23,7 +23,7 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) -from fdroidserver import common, index, nightly +from fdroidserver import common, exception, index, nightly DEBUG_KEYSTORE = '/u3+7QAAAAIAAAABAAAAAQAPYW5kcm9pZGRlYnVna2V5AAABNYhAuskAAAK8MIICuDAOBgorBgEEASoCEQEBBQAEggKkqRnFlhidQmVff83bsAeewXPIsF0jiymzJnvrnUAQtCK0MV9uZonu37Mrj/qKLn56mf6QcvEoKvpCstZxzftgYYpAHWMVLM+hy2Z707QZEHlY7Ukppt8DItj+dXkeqGt7f8KzOb2AQwDbt9lm1fJb+MefLowTaubtvrLMcKIne43CbCu2D8HyN7RPWpEkVetA2Qgr5W4sa3tIUT80afqo9jzwJjKCspuxY9A1M8EIM3/kvyLo2B9r0cuWwRjYZXJ6gmTYI2ARNz0KQnCZUok14NDg+mZTb1B7AzRfb0lfjbA6grbzuAL+WaEpO8/LgGfuOh7QBZBT498TElOaFfQ9toQWA79wAmrQCm4OoFukpPIy2m/l6VjJSmlK5Q+CMOl/Au7OG1sUUCTvPaIr0XKnsiwDJ7a71n9garnPWHkvuWapSRCzCNgaUoGQjB+fTMJFFrwT8P1aLfM6onc3KNrDStoQZuYe5ngCLlNS56bENkVGvJBfdkboxtHZjqDXXON9jWGSOI527J3o2D5sjSVyx3T9XPrsL4TA/nBtdU+c/+M6aoASZR2VymzAKdMrGfj9kE5GXp8vv2vkJj9+OJ4Jm5yeczocc/Idtojjb1yg+sq1yY8kAQxgezpY1rpgi2jF3tSN01c23DNvAaSJLJX2ZuH8sD40ACc80Y1Qp1nUTdpwBZUeaeNruBwx4PHU8GnC71FwtiUpwNs0OoSl0pgDUJ3ODC5bs8B5QmW1wu1eg7I4mMSmCsNGW6VN3sFcu+WEqnmTxPoZombdFZKxsr2oq359Nn4bJ6Uc9PBz/sXsns7Zx1vND/oK/Jv5Y269UVAMeKX/eGpfnxzagW3tqGbOu12C2p9Azo5VxiU2fG/tmk2PjaG5hV/ywReco7I6C1p8OWM2fwAAAAEABVguNTA5AAAB6TCCAeUwggFOoAMCAQICBE89gTUwDQYJKoZIhvcNAQEFBQAwNzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0FuZHJvaWQxFjAUBgNVBAMTDUFuZHJvaWQgRGVidWcwHhcNMTIwMjE2MjIyMDM3WhcNNDIwMjA4MjIyMDM3WjA3MQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQW5kcm9pZDEWMBQGA1UEAxMNQW5kcm9pZCBEZWJ1ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA3AKU7S7JXhUjEwxWP1/LPHXieh61SaA/+xbpqsPA+yjGz1sAcGAyuG6bjNAVm56pq7nkjJzicX7Wi83nUBo58DEC/quxOLdy0C4PEOSAeTnTT1RJIwMDvOgiL1GFCErvQ7gCH6zuAID/JRFbN6nIkhDjs2DYnSBl7aJJf8wCLc0CAwEAATANBgkqhkiG9w0BAQUFAAOBgQAoq/TJffA0l+ZGf89xndmHdxrO6qi+TzSlByvLZ4eFfCovTh1iO+Edrd5V1yXGLxyyvdsadMAFZT8SaxMrP5xxhJ0nra0APWYLpA96M//auMhQBWPgqPntwgvEZuEH7f0kdItjBJ39yijbG8xfgwid6XqNUo0TDDkp/wNWKpJ9tJe+2PrGw1NAvrgSydoH2j8DI1Eq' @@ -168,6 +168,33 @@ class NightlyTest(unittest.TestCase): assert (dot_ssh / AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME).exists() assert (dot_ssh / (AOSP_TESTKEY_DEBUG_KEYSTORE_KEY_FILE_NAME + '.pub')).exists() + @patch('fdroidserver.common.vcs_git.git', lambda args, e: common.PopenResult(1)) + @patch('sys.argv', ['fdroid nightly', '--verbose']) + def test_private_or_non_existent_git_mirror(self): + """Test that this exits with an error when the git mirror repo won't work + + Careful! If the test environment is setup wrong, it can mess + up local files in ~/.ssh or ~/.android. + + """ + os.chdir(self.testdir) + with patch.dict( + os.environ, + { + 'CI': 'true', + 'CI_PROJECT_PATH': 'thisshouldneverexist/orthistoo', + 'CI_PROJECT_URL': 'https://gitlab.com/thisshouldneverexist/orthistoo', + 'DEBUG_KEYSTORE': DEBUG_KEYSTORE, + 'GITLAB_USER_NAME': 'username', + 'GITLAB_USER_EMAIL': 'username@example.com', + 'HOME': str(self.testdir), + 'PATH': os.getenv('PATH'), + }, + clear=True, + ): + with self.assertRaises(exception.VCSException): + nightly.main() + def _put_fdroid_in_args(self, args): """Find fdroid command that belongs to this source code tree""" fdroid = os.path.join(localmodule, 'fdroid') From 83335437b9ed7eb6548c7c25c3d3e5faf677b957 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 11:10:50 +0100 Subject: [PATCH 07/13] nightly: if repo is too large, set archive_older to 3 If the user has not manually set --archive-older, then this will auto-switch it from 20 to 3 to shrink the repo down so it fits into GitLab Pages. --- fdroidserver/nightly.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 8e11fa0b..045f5496 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -197,10 +197,11 @@ def main(): default=False, help=_("Don't use rsync checksums"), ) + archive_older_unset = -1 parser.add_argument( "--archive-older", type=int, - default=20, + default=archive_older_unset, help=_("Set maximum releases in repo before older ones are archived"), ) # TODO add --with-btlog @@ -280,9 +281,10 @@ def main(): repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') - git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') - git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') - git_mirror_statsdir = os.path.join(git_mirror_path, 'fdroid', 'stats') + git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid') + git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') + git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) vcs = common.getvcs('git', clone_url, git_mirror_path) @@ -335,6 +337,19 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, with open(ssh_config, 'a') as fp: fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file) + if options.archive_older == archive_older_unset: + fdroid_size = common.get_dir_size(git_mirror_fdroiddir) + max_size = common.GITLAB_COM_PAGES_MAX_SIZE + if fdroid_size < max_size: + options.archive_older = 20 + else: + options.archive_older = 3 + print( + 'WARNING: repo is %s over the GitLab Pages limit (%s)' + % (fdroid_size - max_size, max_size) + ) + print('Setting --archive-older to 3') + config = { 'identity_file': ssh_private_key_file, 'repo_name': repo_git_base, From f24613b70170036ae09d2b7d19a07a8dbef8e2bc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 11:25:28 +0100 Subject: [PATCH 08/13] index: fix git-mirror size check for GitLab Pages The test case had the wrong folder setup, this was confirmed on a production repo setup. --- fdroidserver/index.py | 3 ++- tests/index.TestCase | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index c788509d..500c2e9e 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1438,7 +1438,8 @@ def get_mirror_service_urls(url): segments.extend([branch, folder]) urls.append('/'.join(segments)) elif hostname == "gitlab.com": - if common.get_dir_size(folder) <= common.GITLAB_COM_PAGES_MAX_SIZE: + git_mirror_path = os.path.join('git-mirror', folder) + if common.get_dir_size(git_mirror_path) <= common.GITLAB_COM_PAGES_MAX_SIZE: # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] urls.append('/'.join(gitlab_pages)) diff --git a/tests/index.TestCase b/tests/index.TestCase index b04d81c0..6e63ffcf 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -416,8 +416,9 @@ class IndexTest(unittest.TestCase): def test_gitlab_get_mirror_service_urls(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - os.mkdir('fdroid') - with Path('fdroid/placeholder').open('w') as fp: + git_mirror_path = Path('git-mirror/fdroid') + git_mirror_path.mkdir(parents=True) + with (git_mirror_path / 'placeholder').open('w') as fp: fp.write(' ') for url in [ 'git@gitlab.com:group/project', From cdce0958f8caed7011c5c2815b7de5b33d255956 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 12:39:25 +0100 Subject: [PATCH 09/13] deploy: convert .gitlab-ci.yml generation to dict + yaml.dump() --- fdroidserver/deploy.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 1ca90ab4..e9b30ac6 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -25,6 +25,7 @@ import re import subprocess import time import urllib +import yaml from argparse import ArgumentParser import logging import shutil @@ -445,16 +446,21 @@ def update_servergitmirrors(servergitmirrors, repo_section): continue if remote.name == 'gitlab': logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') - with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file: - out_file.write("""pages: - script: - - mkdir .public - - cp -r * .public/ - - mv .public public - artifacts: - paths: - - public -""") + with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: + yaml.dump( + { + 'pages': { + 'script': [ + 'mkdir .public', + 'cp -r * .public/', + 'mv .public public', + ], + 'artifacts': {'paths': ['public']}, + } + }, + fp, + default_flow_style=False, + ) repo.git.add(all=True) repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") From d0976a3684fcd66a18341ba268fdda1d86b540e9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 12:10:27 +0100 Subject: [PATCH 10/13] deploy: check repo size before enabling GitLab Pages --- fdroidserver/deploy.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index e9b30ac6..69d09784 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -377,7 +377,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): if repo_section == 'repo': git_mirror_path = 'git-mirror' dotgit = os.path.join(git_mirror_path, '.git') - git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section) + git_fdroiddir = os.path.join(git_mirror_path, 'fdroid') + git_repodir = os.path.join(git_fdroiddir, repo_section) if not os.path.isdir(git_repodir): os.makedirs(git_repodir) # github/gitlab use bare git repos, so only count the .git folder @@ -439,6 +440,18 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: progress = None + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + # push for every remote. This will overwrite the git history for remote in repo.remotes: if remote.name not in enabled_remotes: @@ -449,7 +462,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: yaml.dump( { - 'pages': { + gitlab_ci_job_name: { 'script': [ 'mkdir .public', 'cp -r * .public/', From 947d94e0a9f7f42977beb42d76a73b6d83a6ef46 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 14:37:08 +0100 Subject: [PATCH 11/13] deploy: support GitLab Job Artifacts as a mirror --- fdroidserver/index.py | 20 ++++++++++++++++++++ tests/index.TestCase | 34 +++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 500c2e9e..59610d53 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1453,6 +1453,26 @@ def get_mirror_service_urls(url): # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder" gitlab_raw = segments + ['-', 'raw', branch, folder] urls.append('/'.join(gitlab_raw)) + # GitLab Artifacts "https://user.gitlab.io/-/repo/-/jobs/job_id/artifacts/public/folder" + job_id = os.getenv('CI_JOB_ID') + try: + int(job_id) + gitlab_artifacts = [ + "https:", + "", + user + ".gitlab.io", + '-', + repo, + '-', + 'jobs', + job_id, + 'artifacts', + 'public', + folder, + ] + urls.append('/'.join(gitlab_artifacts)) + except (TypeError, ValueError): + pass # no Job ID to use, ignore return urls diff --git a/tests/index.TestCase b/tests/index.TestCase index 6e63ffcf..1c8e9854 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -414,10 +414,16 @@ class IndexTest(unittest.TestCase): fdroidserver.index.get_mirror_service_urls(url), ) + @patch.dict(os.environ, clear=True) def test_gitlab_get_mirror_service_urls(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): git_mirror_path = Path('git-mirror/fdroid') git_mirror_path.mkdir(parents=True) + ci_job_id = '12345678' + artifacts_url = ( + 'https://group.gitlab.io/-/project/-/jobs/%s/artifacts/public/fdroid' + % ci_job_id + ) with (git_mirror_path / 'placeholder').open('w') as fp: fp.write(' ') for url in [ @@ -427,20 +433,34 @@ class IndexTest(unittest.TestCase): 'https://gitlab.com/group/project.git', ]: with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 1000): + expected = [ + 'https://group.gitlab.io/project/fdroid', + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ] self.assertEqual( - [ - 'https://group.gitlab.io/project/fdroid', - 'https://gitlab.com/group/project/-/raw/master/fdroid', - ], + expected, fdroidserver.index.get_mirror_service_urls(url), ) + with patch.dict(os.environ, clear=True): + os.environ['CI_JOB_ID'] = ci_job_id + self.assertEqual( + expected + [artifacts_url], + fdroidserver.index.get_mirror_service_urls(url), + ) with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10): + expected = [ + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ] self.assertEqual( - [ - 'https://gitlab.com/group/project/-/raw/master/fdroid', - ], + expected, fdroidserver.index.get_mirror_service_urls(url), ) + with patch.dict(os.environ, clear=True): + os.environ['CI_JOB_ID'] = ci_job_id + self.assertEqual( + expected + [artifacts_url], + fdroidserver.index.get_mirror_service_urls(url), + ) def test_make_website(self): tmptestsdir = tempfile.mkdtemp( From 1a30766c24caef4873329beddbd783fb84d29754 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 15 Nov 2022 14:50:23 +0100 Subject: [PATCH 12/13] deploy: GIT_DEPTH=1 so GitLab Pages job runs as quick as possible * https://docs.gitlab.com/ee/ci/large_repositories/index.html#shallow-cloning --- fdroidserver/deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 69d09784..13a8982b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -469,6 +469,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): 'mv .public public', ], 'artifacts': {'paths': ['public']}, + 'variables': {'GIT_DEPTH': 1}, } }, fp, From 09b0405eb01b39e7c293d5fc36384873ce762eef Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 14 Sep 2022 23:48:12 +0200 Subject: [PATCH 13/13] gitlab-ci: apksigner from current build-tools in ubuntu_jammy_pip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20a82096..8c21edf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ ubuntu_jammy_pip: # back to bare machine to act as user's install machine - export ANDROID_HOME=/opt/android-sdk - $pip install sdkmanager - - sdkmanager 'build-tools;30.0.0' + - sdkmanager 'build-tools;33.0.0' - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz