mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-11 01:40:11 +01:00
fix(deploy): remove the index-only mode from elsewhere except the config file
To enable per-remote configuration
This commit is contained in:
proletarius101
Hans-Christoph Steiner
parent
15c8926e75
commit
4c15cea78a
@ -264,7 +264,7 @@ __complete_nightly() {
|
||||
__complete_deploy() {
|
||||
opts="-i -v -q"
|
||||
lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
|
||||
--verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
|
||||
--verbose --quiet --no-checksum --no-keep-git-mirror-archive"
|
||||
__complete_options
|
||||
}
|
||||
|
||||
|
||||
@ -4301,7 +4301,7 @@ def get_app_display_name(app):
|
||||
return app.get('AutoName') or app['id']
|
||||
|
||||
|
||||
def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
|
||||
def local_rsync(options, from_paths: List[str], todir: str):
|
||||
"""Rsync method for local to local copying of things.
|
||||
|
||||
This is an rsync wrapper with all the settings for safe use within
|
||||
@ -4318,7 +4318,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []
|
||||
rsyncargs += ['--verbose']
|
||||
if options.quiet:
|
||||
rsyncargs += ['--quiet']
|
||||
rsyncargs += args
|
||||
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
|
||||
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
@ -92,7 +92,7 @@ def _get_index_includes(base_dir):
|
||||
return index_includes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section, verbose=False, quiet=False):
|
||||
def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False):
|
||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||
|
||||
The contents of that subdir of the
|
||||
@ -112,26 +112,26 @@ def update_awsbucket(repo_section, verbose=False, quiet=False):
|
||||
logging.warning(
|
||||
'No syncing tool set in config.yml!. Defaulting to using s3cmd'
|
||||
)
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
if config['s3cmd'] is True and config['rclone'] is True:
|
||||
logging.warning(
|
||||
'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
|
||||
)
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
if config['s3cmd'] is True and config['rclone'] is not True:
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
if config['rclone'] is True and config['s3cmd'] is not True:
|
||||
update_remote_storage_with_rclone(repo_section, verbose, quiet)
|
||||
|
||||
elif common.set_command_in_config('s3cmd'):
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
elif common.set_command_in_config('rclone'):
|
||||
update_remote_storage_with_rclone(repo_section, verbose, quiet)
|
||||
else:
|
||||
update_awsbucket_libcloud(repo_section)
|
||||
update_awsbucket_libcloud(repo_section, index_only)
|
||||
|
||||
|
||||
def update_awsbucket_s3cmd(repo_section):
|
||||
def update_awsbucket_s3cmd(repo_section, index_only=False):
|
||||
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
||||
|
||||
The upload is done in multiple passes to reduce the chance of
|
||||
@ -185,7 +185,7 @@ def update_awsbucket_s3cmd(repo_section):
|
||||
)
|
||||
)
|
||||
|
||||
if options.index_only:
|
||||
if index_only:
|
||||
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||
.format(path=repo_section, url=s3url))
|
||||
sync_indexes_flags = []
|
||||
@ -352,7 +352,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_awsbucket_libcloud(repo_section):
|
||||
def update_awsbucket_libcloud(repo_section, index_only=False):
|
||||
"""No summary.
|
||||
|
||||
Upload the contents of the directory `repo_section` (including
|
||||
@ -400,7 +400,7 @@ def update_awsbucket_libcloud(repo_section):
|
||||
if obj.name.startswith(upload_dir + '/'):
|
||||
objs[obj.name] = obj
|
||||
|
||||
if options.index_only:
|
||||
if index_only:
|
||||
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||
@ -502,8 +502,9 @@ def update_serverwebroot(serverwebroot, repo_section):
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'],
|
||||
]
|
||||
url = serverwebroot['url']
|
||||
index_only = serverwebroot.get('index_only', False)
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||
if options.index_only:
|
||||
if index_only:
|
||||
rsyncargs += _get_index_file_paths(repo_section)
|
||||
rsyncargs += [f'{url}/{repo_section}/']
|
||||
logging.info(rsyncargs)
|
||||
@ -563,8 +564,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
|
||||
push to all the servers that are configured.
|
||||
|
||||
"""
|
||||
if options.index_only:
|
||||
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
|
||||
logging.info('Syncing from local_copy_dir to this repo.')
|
||||
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
@ -588,8 +588,6 @@ def update_localcopy(repo_section, local_copy_dir):
|
||||
drive.
|
||||
|
||||
"""
|
||||
if options.index_only:
|
||||
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
|
||||
|
||||
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
||||
common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
|
||||
@ -662,12 +660,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
|
||||
shutil.rmtree(archive_path, ignore_errors=True)
|
||||
|
||||
if options.index_only:
|
||||
# rsync is very particular about trailing slashes
|
||||
common.local_rsync(common.get_options(),
|
||||
_get_index_file_paths(repo_section),
|
||||
git_repodir.rstrip('/') + '/')
|
||||
else:
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
common.local_rsync(common.get_options(),
|
||||
@ -681,10 +673,24 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
elif 'identity_file' in config:
|
||||
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
|
||||
|
||||
if options.verbose:
|
||||
progressbar = progress.Bar()
|
||||
|
||||
class MyProgressPrinter(git.RemoteProgress):
|
||||
def update(self, op_code, current, maximum=None, message=None):
|
||||
if isinstance(maximum, float):
|
||||
progressbar.show(current, maximum)
|
||||
|
||||
progress = MyProgressPrinter()
|
||||
else:
|
||||
progress = None
|
||||
|
||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||
initial_commit_ref = repo.head.ref
|
||||
|
||||
enabled_remotes = []
|
||||
for d in servergitmirrors:
|
||||
index_only = d.get('index_only', False)
|
||||
remote_url = d['url']
|
||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
enabled_remotes.append(name)
|
||||
@ -697,24 +703,17 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
repo.create_remote(name, remote_url)
|
||||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
if index_only:
|
||||
logging.debug('Adding index files to git mirror')
|
||||
repo.index.add(_get_index_file_paths(repo_section))
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
repo.git.add(all=True)
|
||||
logging.debug('Committing all files into git mirror')
|
||||
|
||||
logging.debug('Committing files into git mirror')
|
||||
repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
if options.verbose:
|
||||
progressbar = progress.Bar()
|
||||
|
||||
class MyProgressPrinter(git.RemoteProgress):
|
||||
def update(self, op_code, current, maximum=None, message=None):
|
||||
if isinstance(maximum, float):
|
||||
progressbar.show(current, maximum)
|
||||
|
||||
progress = MyProgressPrinter()
|
||||
else:
|
||||
progress = None
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
gitlab_ci_job_name = 'pages'
|
||||
@ -727,11 +726,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||
)
|
||||
|
||||
# push for every remote. This will overwrite the git history
|
||||
for remote in repo.remotes:
|
||||
if remote.name not in enabled_remotes:
|
||||
repo.delete_remote(remote)
|
||||
continue
|
||||
# push. This will overwrite the git history
|
||||
remote = repo.remote(name)
|
||||
if remote.name == 'gitlab':
|
||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||
@ -751,7 +747,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
default_flow_style=False,
|
||||
)
|
||||
|
||||
repo.git.add(all=True)
|
||||
repo.index.add(['.gitlab-ci.yml'])
|
||||
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||
|
||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||
@ -780,6 +776,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
else:
|
||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||
|
||||
repo.head.reset(initial_commit_ref, index=True, working_tree=True)
|
||||
|
||||
if progress:
|
||||
progressbar.done()
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
from fdroidserver import index
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||
@ -72,7 +73,6 @@ class DeployTest(unittest.TestCase):
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk0 = url0 / fake_apk
|
||||
@ -159,7 +159,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
@ -187,7 +186,6 @@ class DeployTest(unittest.TestCase):
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
@ -195,7 +193,9 @@ class DeployTest(unittest.TestCase):
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
|
||||
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': str(url), 'index_only': True}, 'repo'
|
||||
)
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
@ -310,7 +310,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
@ -370,34 +369,6 @@ class DeployTest(unittest.TestCase):
|
||||
'example.com:/var/www/fdroid',
|
||||
],
|
||||
)
|
||||
# elif call_iteration == 1:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--quiet',
|
||||
# 'repo',
|
||||
# serverwebroot,
|
||||
# ],
|
||||
# )
|
||||
# elif call_iteration == 2:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--quiet',
|
||||
# 'Sym.apk',
|
||||
# 'Sym.apk.asc',
|
||||
# 'Sym.apk.sig',
|
||||
# 'example.com:/var/www/fdroid',
|
||||
# ],
|
||||
# )
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
@ -409,7 +380,9 @@ class DeployTest(unittest.TestCase):
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': url, 'index_only': True}, repo_section
|
||||
)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
def test_update_serverwebroot_with_id_file(self):
|
||||
@ -499,7 +472,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.verbose = True
|
||||
fdroidserver.deploy.options.quiet = False
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
url = "example.com:/var/www/fdroid"
|
||||
@ -550,29 +522,15 @@ class DeployTest(unittest.TestCase):
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
# elif call_iteration == 1:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--verbose',
|
||||
# '-e',
|
||||
# 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
# + fdroidserver.deploy.config['identity_file'],
|
||||
# 'archive',
|
||||
# serverwebroot,
|
||||
# ],
|
||||
# )
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': url, 'index_only': True}, repo_section
|
||||
)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
@unittest.skipIf(
|
||||
@ -601,7 +559,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -733,7 +690,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -807,7 +763,9 @@ class DeployTest(unittest.TestCase):
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(
|
||||
repo_section, index_only=True
|
||||
)
|
||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||
|
||||
def test_update_awsbucket_libcloud(self):
|
||||
@ -818,7 +776,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -880,7 +837,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -915,7 +871,7 @@ class DeployTest(unittest.TestCase):
|
||||
mock_driver.get_container.return_value = mock_container
|
||||
mock_driver.upload_object_via_stream.return_value = None
|
||||
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
|
||||
|
||||
mock_driver.get_container.assert_called_once_with(
|
||||
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||
@ -944,7 +900,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -1001,7 +956,6 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
@ -1038,16 +992,7 @@ class DeployTest(unittest.TestCase):
|
||||
'--delete',
|
||||
'--chmod=Da+rx,Fa-x,a+r,u+w',
|
||||
'--quiet',
|
||||
'repo/entry.jar',
|
||||
'repo/entry.json',
|
||||
'repo/entry.json.asc',
|
||||
'repo/index-v1.jar',
|
||||
'repo/index-v1.json',
|
||||
'repo/index-v1.json.asc',
|
||||
'repo/index-v2.json',
|
||||
'repo/index-v2.json.asc',
|
||||
'repo/index.jar',
|
||||
'repo/index.xml',
|
||||
'repo/',
|
||||
"git-mirror/fdroid/repo/",
|
||||
],
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user