1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 12:20:12 +02:00

fix(deploy): remove the index-only mode from elsewhere except the config file

To enable per-remote configuration
This commit is contained in:
proletarius101 2024-02-26 22:57:05 +08:00 committed by Hans-Christoph Steiner
parent 15c8926e75
commit 4c15cea78a
4 changed files with 80 additions and 138 deletions

View File

@ -264,7 +264,7 @@ __complete_nightly() {
__complete_deploy() {
opts="-i -v -q"
lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
--verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
--verbose --quiet --no-checksum --no-keep-git-mirror-archive"
__complete_options
}

View File

@ -4301,7 +4301,7 @@ def get_app_display_name(app):
return app.get('AutoName') or app['id']
def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
def local_rsync(options, from_paths: List[str], todir: str):
"""Rsync method for local to local copying of things.
This is an rsync wrapper with all the settings for safe use within
@ -4318,7 +4318,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []
rsyncargs += ['--verbose']
if options.quiet:
rsyncargs += ['--quiet']
rsyncargs += args
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
raise FDroidException()

View File

@ -92,7 +92,7 @@ def _get_index_includes(base_dir):
return index_includes
def update_awsbucket(repo_section, verbose=False, quiet=False):
def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False):
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
The contents of that subdir of the
@ -112,26 +112,26 @@ def update_awsbucket(repo_section, verbose=False, quiet=False):
logging.warning(
'No syncing tool set in config.yml!. Defaulting to using s3cmd'
)
update_awsbucket_s3cmd(repo_section)
update_awsbucket_s3cmd(repo_section, index_only)
if config['s3cmd'] is True and config['rclone'] is True:
logging.warning(
'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
)
update_awsbucket_s3cmd(repo_section)
update_awsbucket_s3cmd(repo_section, index_only)
if config['s3cmd'] is True and config['rclone'] is not True:
update_awsbucket_s3cmd(repo_section)
update_awsbucket_s3cmd(repo_section, index_only)
if config['rclone'] is True and config['s3cmd'] is not True:
update_remote_storage_with_rclone(repo_section, verbose, quiet)
elif common.set_command_in_config('s3cmd'):
update_awsbucket_s3cmd(repo_section)
update_awsbucket_s3cmd(repo_section, index_only)
elif common.set_command_in_config('rclone'):
update_remote_storage_with_rclone(repo_section, verbose, quiet)
else:
update_awsbucket_libcloud(repo_section)
update_awsbucket_libcloud(repo_section, index_only)
def update_awsbucket_s3cmd(repo_section):
def update_awsbucket_s3cmd(repo_section, index_only=False):
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
The upload is done in multiple passes to reduce the chance of
@ -185,7 +185,7 @@ def update_awsbucket_s3cmd(repo_section):
)
)
if options.index_only:
if index_only:
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
.format(path=repo_section, url=s3url))
sync_indexes_flags = []
@ -352,7 +352,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
raise FDroidException()
def update_awsbucket_libcloud(repo_section):
def update_awsbucket_libcloud(repo_section, index_only=False):
"""No summary.
Upload the contents of the directory `repo_section` (including
@ -400,7 +400,7 @@ def update_awsbucket_libcloud(repo_section):
if obj.name.startswith(upload_dir + '/'):
objs[obj.name] = obj
if options.index_only:
if index_only:
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
files_to_upload = list(set(files_to_upload) & set(index_files))
@ -502,8 +502,9 @@ def update_serverwebroot(serverwebroot, repo_section):
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'],
]
url = serverwebroot['url']
index_only = serverwebroot.get('index_only', False)
logging.info('rsyncing ' + repo_section + ' to ' + url)
if options.index_only:
if index_only:
rsyncargs += _get_index_file_paths(repo_section)
rsyncargs += [f'{url}/{repo_section}/']
logging.info(rsyncargs)
@ -563,8 +564,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
push to all the servers that are configured.
"""
if options.index_only:
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
logging.info('Syncing from local_copy_dir to this repo.')
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
@ -588,8 +588,6 @@ def update_localcopy(repo_section, local_copy_dir):
drive.
"""
if options.index_only:
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
# local_copy_dir is guaranteed to have a trailing slash in main() below
common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
@ -662,17 +660,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
shutil.rmtree(archive_path, ignore_errors=True)
if options.index_only:
# rsync is very particular about trailing slashes
common.local_rsync(common.get_options(),
_get_index_file_paths(repo_section),
git_repodir.rstrip('/') + '/')
else:
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
common.local_rsync(common.get_options(),
[repo_section.rstrip('/') + '/'],
git_repodir.rstrip('/') + '/')
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
common.local_rsync(common.get_options(),
[repo_section.rstrip('/') + '/'],
git_repodir.rstrip('/') + '/')
# use custom SSH command if identity_file specified
ssh_cmd = 'ssh -oBatchMode=yes'
@ -681,28 +673,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
elif 'identity_file' in config:
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
enabled_remotes = []
for d in servergitmirrors:
remote_url = d['url']
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
enabled_remotes.append(name)
r = git.remote.Remote(repo, name)
if r in repo.remotes:
r = repo.remote(name)
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
r.set_url(remote_url)
else:
repo.create_remote(name, remote_url)
logging.info('Mirroring to: ' + remote_url)
# sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror')
repo.git.add(all=True)
logging.debug('Committing all files into git mirror')
repo.index.commit("fdroidserver git-mirror")
if options.verbose:
progressbar = progress.Bar()
@ -715,23 +685,49 @@ def update_servergitmirrors(servergitmirrors, repo_section):
else:
progress = None
# only deploy to GitLab Artifacts if too big for GitLab Pages
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
gitlab_ci_job_name = 'pages'
else:
gitlab_ci_job_name = 'GitLab Artifacts'
logging.warning(
_(
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
)
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
)
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
initial_commit_ref = repo.head.ref
# push for every remote. This will overwrite the git history
for remote in repo.remotes:
if remote.name not in enabled_remotes:
repo.delete_remote(remote)
continue
enabled_remotes = []
for d in servergitmirrors:
index_only = d.get('index_only', False)
remote_url = d['url']
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
enabled_remotes.append(name)
r = git.remote.Remote(repo, name)
if r in repo.remotes:
r = repo.remote(name)
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
r.set_url(remote_url)
else:
repo.create_remote(name, remote_url)
logging.info('Mirroring to: ' + remote_url)
if index_only:
logging.debug('Adding index files to git mirror')
repo.index.add(_get_index_file_paths(repo_section))
else:
# sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror')
repo.git.add(all=True)
logging.debug('Committing files into git mirror')
repo.index.commit("fdroidserver git-mirror")
# only deploy to GitLab Artifacts if too big for GitLab Pages
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
gitlab_ci_job_name = 'pages'
else:
gitlab_ci_job_name = 'GitLab Artifacts'
logging.warning(
_(
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
)
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
)
# push. This will overwrite the git history
remote = repo.remote(name)
if remote.name == 'gitlab':
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
@ -751,7 +747,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
default_flow_style=False,
)
repo.git.add(all=True)
repo.index.add(['.gitlab-ci.yml'])
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
logging.debug(_('Pushing to {url}').format(url=remote.url))
@ -780,6 +776,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
else:
logging.debug(remote.url + ': ' + pushinfo.summary)
repo.head.reset(initial_commit_ref, index=True, working_tree=True)
if progress:
progressbar.done()

View File

@ -10,6 +10,7 @@ import tempfile
import unittest
from pathlib import Path
from unittest import mock
from fdroidserver import index
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
@ -72,7 +73,6 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.index_only = False
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk0 = url0 / fake_apk
@ -159,7 +159,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.index_only = False
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk
@ -187,7 +186,6 @@ class DeployTest(unittest.TestCase):
# setup parameters for this test run
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.index_only = True
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk
@ -195,7 +193,9 @@ class DeployTest(unittest.TestCase):
self.assertFalse(dest_apk.is_file())
self.assertFalse(dest_index.is_file())
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
fdroidserver.deploy.update_serverwebroot(
{'url': str(url), 'index_only': True}, 'repo'
)
self.assertFalse(dest_apk.is_file())
self.assertTrue(dest_index.is_file())
@ -310,7 +310,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.index_only = True
fdroidserver.deploy.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid"
repo_section = 'repo'
@ -370,34 +369,6 @@ class DeployTest(unittest.TestCase):
'example.com:/var/www/fdroid',
],
)
# elif call_iteration == 1:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--quiet',
# 'repo',
# serverwebroot,
# ],
# )
# elif call_iteration == 2:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--quiet',
# 'Sym.apk',
# 'Sym.apk.asc',
# 'Sym.apk.sig',
# 'example.com:/var/www/fdroid',
# ],
# )
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
@ -409,7 +380,9 @@ class DeployTest(unittest.TestCase):
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
fdroidserver.deploy.update_serverwebroot(
{'url': url, 'index_only': True}, repo_section
)
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
def test_update_serverwebroot_with_id_file(self):
@ -499,7 +472,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.verbose = True
fdroidserver.deploy.options.quiet = False
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.index_only = True
fdroidserver.deploy.config['identity_file'] = './id_rsa'
fdroidserver.deploy.config['make_current_version_link'] = False
url = "example.com:/var/www/fdroid"
@ -550,29 +522,15 @@ class DeployTest(unittest.TestCase):
"example.com:/var/www/fdroid/archive/",
],
)
# elif call_iteration == 1:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--verbose',
# '-e',
# 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
# + fdroidserver.deploy.config['identity_file'],
# 'archive',
# serverwebroot,
# ],
# )
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
fdroidserver.deploy.update_serverwebroot(
{'url': url, 'index_only': True}, repo_section
)
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
@unittest.skipIf(
@ -601,7 +559,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -733,7 +690,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = True
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -807,7 +763,9 @@ class DeployTest(unittest.TestCase):
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
fdroidserver.deploy.update_awsbucket_s3cmd(
repo_section, index_only=True
)
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
def test_update_awsbucket_libcloud(self):
@ -818,7 +776,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -880,7 +837,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = True
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -915,7 +871,7 @@ class DeployTest(unittest.TestCase):
mock_driver.get_container.return_value = mock_container
mock_driver.upload_object_via_stream.return_value = None
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
mock_driver.get_container.assert_called_once_with(
container_name=fdroidserver.deploy.config["awsbucket"]
@ -944,7 +900,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -1001,7 +956,6 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = True
config = {}
fdroidserver.common.fill_config_defaults(config)
@ -1038,16 +992,7 @@ class DeployTest(unittest.TestCase):
'--delete',
'--chmod=Da+rx,Fa-x,a+r,u+w',
'--quiet',
'repo/entry.jar',
'repo/entry.json',
'repo/entry.json.asc',
'repo/index-v1.jar',
'repo/index-v1.json',
'repo/index-v1.json.asc',
'repo/index-v2.json',
'repo/index-v2.json.asc',
'repo/index.jar',
'repo/index.xml',
'repo/',
"git-mirror/fdroid/repo/",
],
)