From 947217549afeb0bee8b34c927bc86e9372ed72df Mon Sep 17 00:00:00 2001 From: proletarius101 Date: Wed, 14 Feb 2024 16:50:39 +0000 Subject: [PATCH] feat: add servergitmirrors as a dict support --- .gitlab-ci.yml | 2 +- examples/config.yml | 8 ++--- fdroidserver/common.py | 6 ++-- fdroidserver/deploy.py | 21 ++++++++--- fdroidserver/index.py | 3 +- fdroidserver/nightly.py | 2 +- tests/deploy.TestCase | 80 +++++++++++++++++++++++++++++++++++++++++ tests/index.TestCase | 10 +++--- tests/nightly.TestCase | 6 ++-- tests/run-tests | 2 +- 10 files changed, 118 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 866a2ec8..7e7f0b25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -542,7 +542,7 @@ servergitmirrors: - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ - cd /tmp/fdroid - touch fdroid-icon.png - - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py + - printf "servergitmirrors:\n-\ url:\ $SERVER_GIT_MIRROR\n" >> config.yml - $PYTHONPATH/fdroid update --verbose --create-metadata - $PYTHONPATH/fdroid deploy --verbose - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json` diff --git a/examples/config.yml b/examples/config.yml index b094a032..79f020a6 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -198,14 +198,14 @@ # deploy_process_logs: true # The full URL to a git remote repository. You can include -# multiple servers to mirror to by wrapping the whole thing in {} or [], and -# including the servergitmirrors strings in a comma-separated list. +# multiple servers to mirror to by adding strings to a YAML list or map. # Servers listed here will also be automatically inserted in the mirrors list. # # servergitmirrors: https://github.com/user/repo # servergitmirrors: -# - https://github.com/user/repo -# - https://gitlab.com/user/repo +# - url: https://github.com/user/repo +# - url: https://gitlab.com/user/repo +# indexOnly: true # Most git hosting services have hard size limits for each git repo. # `fdroid deploy` will delete the git history when the git mirror repo diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3da0a193..e8774ef5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -67,7 +67,7 @@ from pyasn1.error import PyAsn1Error import fdroidserver.metadata import fdroidserver.lint from fdroidserver import _ -from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\ +from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException, \ BuildException, VerificationException, MetaDataException from .asynchronousfilereader import AsynchronousFileReader from .looseversion import LooseVersion @@ -482,8 +482,10 @@ def read_config(opts=None): if 'servergitmirrors' in config: if isinstance(config['servergitmirrors'], str): - roots = [config['servergitmirrors']] + roots = [{"url": config['servergitmirrors']}] elif all(isinstance(item, str) for item in config['servergitmirrors']): + roots = [{'url': i} for i in config['servergitmirrors']] + elif all(isinstance(item, dict) for item in config['servergitmirrors']): roots = config['servergitmirrors'] else: raise TypeError(_('only accepts strings, lists, and tuples')) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 92287f1b..2f8c4569 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -47,6 +47,19 @@ AUTO_S3CFG = '.fdroid-deploy-s3cfg' USER_S3CFG = 's3cfg' REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') +INDEX_FILES = [ + "entry.jar", + "entry.json", + "entry.json.asc", + "index-v1.jar", + "index-v1.json", + "index-v1.json.asc", + "index-v2.json", + "index-v2.json.asc", + "index.jar", + "index.xml", +] + def _get_index_excludes(repo_section): """Return the list of files to be synced last, since they finalize the deploy. @@ -447,7 +460,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) enabled_remotes = [] - for remote_url in servergitmirrors: + for d in servergitmirrors: + remote_url = d['url'] name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) enabled_remotes.append(name) r = git.remote.Remote(repo, name) @@ -840,10 +854,9 @@ def main(): update_serverwebroots( config['serverwebroot'], repo_section, standardwebroot ) - if config.get('servergitmirrors', []): + if config.get('servergitmirrors'): # update_servergitmirrors will take care of multiple mirrors so don't need a foreach - servergitmirrors = config.get('servergitmirrors', []) - update_servergitmirrors(servergitmirrors, repo_section) + update_servergitmirrors(config['servergitmirrors'], repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) if config.get('androidobservatory'): diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 9a631eb8..f64b8806 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1478,7 +1478,7 @@ def add_mirrors_to_repodict(repo_section, repodict): repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']}) -def get_mirror_service_urls(url): +def get_mirror_service_urls(mirror): """Get direct URLs from git service for use by fdroidclient. Via 'servergitmirrors', fdroidserver can create and push a mirror @@ -1496,6 +1496,7 @@ def get_mirror_service_urls(url): information about the repo available to end user. """ + url = mirror['url'] if url.startswith('git@'): url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 2d1c1ead..2d0b2325 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -358,7 +358,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, 'archive_url': repo_base + '/archive', 'archive_description': 'Old nightly builds that have been archived.', 'archive_older': options.archive_older, - 'servergitmirrors': servergitmirror, + 'servergitmirrors': [{"url": servergitmirror}], 'keystore': KEYSTORE_FILE, 'repo_keyalias': KEY_ALIAS, 'keystorepass': PASSWORD, diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index e4334725..bbb0e929 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -300,6 +300,86 @@ class DeployTest(unittest.TestCase): name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) ) + def test_update_servergitmirrors(self): + # setup parameters for this test run + fdroidserver.deploy.options = mock.Mock() + fdroidserver.deploy.options.identity_file = None + fdroidserver.deploy.options.no_keep_git_mirror_archive = False + fdroidserver.deploy.options.verbose = False + fdroidserver.deploy.options.quiet = True + fdroidserver.deploy.options.index_only = False + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + fdroidserver.deploy.config["servergitmirrors"] = [] + + repo_section = 'repo' + + # setup function for asserting subprocess.call invocations + update_servergitmirrors_call_iteration = 0 + remote_push_call_iteration = 0 + + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True) + fake_apk = repo / 'Sym.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + + def update_servergitmirrors_call(cmd): + nonlocal update_servergitmirrors_call_iteration + if update_servergitmirrors_call_iteration == 0: + self.assertListEqual( + cmd, + [ + 'rsync', + '--recursive', + '--safe-links', + '--times', + '--perms', + '--one-file-system', + '--delete', + '--chmod=Da+rx,Fa-x,a+r,u+w', + '--quiet', + 'repo/', + "git-mirror/fdroid/repo/", + ], + ) + else: + self.fail('unexpected subprocess.call invocation') + update_servergitmirrors_call_iteration += 1 + return 0 + + def remote_push_call(ref, force=False, set_upstream=False, **_args): + nonlocal remote_push_call_iteration + if remote_push_call_iteration == 0: + self.assertEqual([ref, force, set_upstream], ['master', True, True]) + else: + self.fail('unexpected git.Remote.push invocation') + remote_push_call_iteration += 1 + return [] + + with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call): + with mock.patch( + 'git.Remote.push', side_effect=remote_push_call + ) as mock_remote_push: + mock_remote_push.return_value = [] + fdroidserver.deploy.update_servergitmirrors( + [{'url': 'https://github.com/user/repo'}], repo_section + ) + self.assertEqual( + update_servergitmirrors_call_iteration, + 1, + 'expected 1 invocations of subprocess.call', + ) + self.assertEqual( + remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push' + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/index.TestCase b/tests/index.TestCase index 00c8474b..8eccec76 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -575,7 +575,7 @@ class IndexTest(unittest.TestCase): ]: self.assertEqual( ['https://raw.githubusercontent.com/foo/bar/master/fdroid'], - index.get_mirror_service_urls(url), + index.get_mirror_service_urls({"url": url}), ) @patch.dict(os.environ, clear=True) @@ -603,13 +603,13 @@ class IndexTest(unittest.TestCase): ] self.assertEqual( expected, - index.get_mirror_service_urls(url), + index.get_mirror_service_urls({"url": url}), ) with patch.dict(os.environ, clear=True): os.environ['CI_JOB_ID'] = ci_job_id self.assertEqual( expected + [artifacts_url], - index.get_mirror_service_urls(url), + index.get_mirror_service_urls({"url": url}), ) with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10): expected = [ @@ -617,13 +617,13 @@ class IndexTest(unittest.TestCase): ] self.assertEqual( expected, - index.get_mirror_service_urls(url), + index.get_mirror_service_urls({"url": url}), ) with patch.dict(os.environ, clear=True): os.environ['CI_JOB_ID'] = ci_job_id self.assertEqual( expected + [artifacts_url], - index.get_mirror_service_urls(url), + index.get_mirror_service_urls({"url": url}), ) def test_make_website(self): diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index 9c861fcb..b6d47447 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -256,7 +256,7 @@ class NightlyTest(unittest.TestCase): self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) self.assertFalse(os.path.exists('config.py')) git_url = 'git@github.com:f-droid/test-nightly' - mirror_url = index.get_mirror_service_urls(git_url)[0] + mirror_url = index.get_mirror_service_urls({"url": git_url})[0] expected = { 'archive_description': 'Old nightly builds that have been archived.', 'archive_name': 'f-droid/test-nightly archive', @@ -271,7 +271,7 @@ class NightlyTest(unittest.TestCase): 'repo_keyalias': 'androiddebugkey', 'repo_name': 'f-droid/test-nightly', 'repo_url': mirror_url + '/repo', - 'servergitmirrors': git_url, + 'servergitmirrors': [{"url": git_url}], 'update_stats': True, } with open('config.yml') as fp: @@ -344,7 +344,7 @@ class NightlyTest(unittest.TestCase): 'repo_keyalias': 'androiddebugkey', 'repo_name': 'fdroid/test-nightly', 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', - 'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly', + 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}], 'update_stats': True, } with open('config.yml') as fp: diff --git a/tests/run-tests b/tests/run-tests index 0dd750d8..ca9aa951 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1154,7 +1154,7 @@ GIT_MIRROR=$REPOROOT/git-mirror cd $REPOROOT fdroid_init_with_prebuilt_keystore printf '\narchive_older: 3\n' >> config.yml -echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml +printf "servergitmirrors: $SERVER_GIT_MIRROR\n" >> config.yml cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ $fdroid update --create-metadata