From 7a656d45e3d26bb1888f7018a33847b04b967163 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 Jan 2024 21:58:12 +0100 Subject: [PATCH] config: convert serverwebroot: to list-of-dicts format This allows for more metadata about the server and deploy mode. --- examples/config.yml | 6 ++++++ fdroidserver/common.py | 13 +++++++++---- fdroidserver/deploy.py | 9 +++++---- tests/common.TestCase | 18 +++++++++++++----- tests/deploy.TestCase | 22 +++++++++++----------- 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 12f7d138..59453376 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -178,6 +178,12 @@ # serverwebroot: # - foo.com:/usr/share/nginx/www/fdroid # - bar.info:/var/www/fdroid +# +# There is a special mode to only deploy the index file: +# +# serverwebroot: +# - url: 'me@b.az:/srv/fdroid' +# indexOnly: true # When running fdroid processes on a remote server, it is possible to diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e97cedf5..3da0a193 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -462,18 +462,22 @@ def read_config(opts=None): if 'serverwebroot' in config: if isinstance(config['serverwebroot'], str): - roots = [config['serverwebroot']] + roots = [{'url': config['serverwebroot']}] elif all(isinstance(item, str) for item in config['serverwebroot']): + roots = [{'url': i} for i in config['serverwebroot']] + elif all(isinstance(item, dict) for item in config['serverwebroot']): roots = config['serverwebroot'] else: raise TypeError(_('only accepts strings, lists, and tuples')) rootlist = [] - for rootstr in roots: + for d in roots: # since this is used with rsync, where trailing slashes have # meaning, ensure there is always a trailing slash + rootstr = d['url'] if rootstr[-1] != '/': rootstr += '/' - rootlist.append(rootstr.replace('//', '/')) + d['url'] = rootstr.replace('//', '/') + rootlist.append(d) config['serverwebroot'] = rootlist if 'servergitmirrors' in config: @@ -4052,7 +4056,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None): logging.debug(_('skip deploying full build logs: not enabled in config')) return - for webroot in config.get('serverwebroot', []): + for d in config.get('serverwebroot', []): + webroot = d['url'] cmd = ['rsync', '--archive', '--delete-after', diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 7cc709d6..aef9205b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -294,11 +294,12 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] elif 'identity_file' in config: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) + url = serverwebroot['url'] + logging.info('rsyncing ' + repo_section + ' to ' + url) excludes = _get_index_excludes(repo_section) - if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: raise FDroidException() - if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + [repo_section, url]) != 0: raise FDroidException() # upload "current version" symlinks if requested if config['make_current_version_link'] and repo_section == 'repo': @@ -308,7 +309,7 @@ def update_serverwebroot(serverwebroot, repo_section): if os.path.islink(f): links_to_upload.append(f) if len(links_to_upload) > 0: - if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0: + if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: raise FDroidException() diff --git a/tests/common.TestCase b/tests/common.TestCase index 68ceee55..55202dcc 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options.quiet = False fdroidserver.common.config = {} fdroidserver.common.config['serverwebroot'] = [ - 'example.com:/var/www/fdroid/', - 'example.com:/var/www/fbot/', + {'url': 'example.com:/var/www/fdroid/'}, + {'url': 'example.com:/var/www/fbot/'}, ] fdroidserver.common.config['deploy_process_logs'] = True fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' @@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.common.config = {} - fdroidserver.common.config['serverwebroot'] = [fakeserver] + fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}] fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' def assert_subprocess_call(cmd): @@ -2872,7 +2872,7 @@ class CommonTest(unittest.TestCase): os.chdir(self.testdir) Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""") self.assertEqual( - ['foo@example.com:/var/www/'], + [{'url': 'foo@example.com:/var/www/'}], fdroidserver.common.read_config()['serverwebroot'], ) @@ -2880,7 +2880,15 @@ class CommonTest(unittest.TestCase): os.chdir(self.testdir) Path('config.yml').write_text("""serverwebroot:\n - foo@example.com:/var/www""") self.assertEqual( - ['foo@example.com:/var/www/'], + [{'url': 'foo@example.com:/var/www/'}], + fdroidserver.common.read_config()['serverwebroot'], + ) + + def test_config_serverwebroot_dict(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot:\n - url: 'foo@example.com:/var/www'""") + self.assertEqual( + [{'url': 'foo@example.com:/var/www/'}], fdroidserver.common.read_config()['serverwebroot'], ) diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 5539af4c..b1f1f103 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -45,16 +45,16 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') - serverwebroot = Path('serverwebroot') - serverwebroot.mkdir() + url = Path('url') + url.mkdir() # setup parameters for this test run fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = False - dest_apk = Path(serverwebroot) / fake_apk + dest_apk = url / fake_apk self.assertFalse(dest_apk.is_file()) - fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo') + fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') self.assertTrue(dest_apk.is_file()) @mock.patch.dict(os.environ, clear=True) @@ -72,7 +72,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.quiet = True fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = True - serverwebroot = "example.com:/var/www/fdroid" + url = "example.com:/var/www/fdroid" repo_section = 'repo' # setup function for asserting subprocess.call invocations @@ -123,7 +123,7 @@ class DeployTest(unittest.TestCase): '--safe-links', '--quiet', 'repo', - serverwebroot, + url, ], ) elif call_iteration == 2: @@ -152,7 +152,7 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call') def test_update_serverwebroot_with_id_file(self): @@ -163,7 +163,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False - serverwebroot = "example.com:/var/www/fdroid" + url = "example.com:/var/www/fdroid" repo_section = 'archive' # setup function for asserting subprocess.call invocations @@ -204,7 +204,7 @@ class DeployTest(unittest.TestCase): '--exclude', 'archive/index.xml', 'archive', - serverwebroot, + url, ], ) elif call_iteration == 1: @@ -220,7 +220,7 @@ class DeployTest(unittest.TestCase): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], 'archive', - serverwebroot, + url, ], ) else: @@ -229,7 +229,7 @@ class DeployTest(unittest.TestCase): return 0 with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') @unittest.skipIf(