kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00
Code Activity

refactor: use _get_index_file_paths() instead of this mix of --include/--exclude

Browse Source
This commit is contained in:
proletarius101 2024-01-20 17:32:33 +08:00
parent c7574037a9
commit 221739b653
No known key found for this signature in database
4 changed files with 85 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
fdroidserver/common.py
View File

@ -4301,7 +4301,7 @@ def get_app_display_name(app):
return app.get('AutoName') or app['id'] return app.get('AutoName') or app['id']
def local_rsync(options, fromdir, todir, args: List[str] = []): def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
"""Rsync method for local to local copying of things. """Rsync method for local to local copying of things.
This is an rsync wrapper with all the settings for safe use within This is an rsync wrapper with all the settings for safe use within
@ -4319,8 +4319,8 @@ def local_rsync(options, fromdir, todir, args: List[str] = []):
if options.quiet: if options.quiet:
rsyncargs += ['--quiet'] rsyncargs += ['--quiet']
rsyncargs += args rsyncargs += args
logging.debug(' '.join(rsyncargs + [fromdir, todir])) logging.debug(' '.join(rsyncargs + from_paths + [todir]))
if subprocess.call(rsyncargs + [fromdir, todir]) != 0: if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
raise FDroidException() raise FDroidException()

17
fdroidserver/deploy.py
View File

@ -504,7 +504,9 @@ def update_serverwebroot(serverwebroot, repo_section):
url = serverwebroot['url'] url = serverwebroot['url']
logging.info('rsyncing ' + repo_section + ' to ' + url) logging.info('rsyncing ' + repo_section + ' to ' + url)
if options.index_only: if options.index_only:
rsyncargs += ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*'] + [repo_section, url] rsyncargs += _get_index_file_paths(repo_section)
rsyncargs += [f'{url}/{repo_section}/']
logging.info(rsyncargs)
if subprocess.call(rsyncargs) != 0: if subprocess.call(rsyncargs) != 0:
raise FDroidException() raise FDroidException()
else: else:
@ -568,7 +570,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
# make sure both paths have exactly one trailing slash # make sure both paths have exactly one trailing slash
common.local_rsync( common.local_rsync(
common.get_options(), common.get_options(),
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/', [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
repo_section.rstrip('/') + '/', repo_section.rstrip('/') + '/',
) )
@ -590,7 +592,7 @@ def update_localcopy(repo_section, local_copy_dir):
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem')) raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
# local_copy_dir is guaranteed to have a trailing slash in main() below # local_copy_dir is guaranteed to have a trailing slash in main() below
common.local_rsync(common.get_options(), repo_section, local_copy_dir) common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR) offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
if os.path.isdir(os.path.join(offline_copy, '.git')): if os.path.isdir(os.path.join(offline_copy, '.git')):
@ -661,18 +663,15 @@ def update_servergitmirrors(servergitmirrors, repo_section):
shutil.rmtree(archive_path, ignore_errors=True) shutil.rmtree(archive_path, ignore_errors=True)
if options.index_only: if options.index_only:
rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
# rsync is very particular about trailing slashes # rsync is very particular about trailing slashes
common.local_rsync(common.get_options(), common.local_rsync(common.get_options(),
repo_section.rstrip('/') + '/', _get_index_file_paths(repo_section),
git_repodir.rstrip('/') + '/', git_repodir.rstrip('/') + '/')
args=rsyncargs)
else: else:
# trailing slashes have a meaning in rsync which is not needed here, so # trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash # make sure both paths have exactly one trailing slash
common.local_rsync(common.get_options(), common.local_rsync(common.get_options(),
repo_section.rstrip('/') + '/', [repo_section.rstrip('/') + '/'],
git_repodir.rstrip('/') + '/') git_repodir.rstrip('/') + '/')
# use custom SSH command if identity_file specified # use custom SSH command if identity_file specified

10
fdroidserver/nightly.py
View File

@ -378,11 +378,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
os.chdir(repo_basedir) os.chdir(repo_basedir)
if os.path.isdir(git_mirror_repodir): if os.path.isdir(git_mirror_repodir):
common.local_rsync(options, git_mirror_repodir + '/', 'repo/') common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
if os.path.isdir(git_mirror_metadatadir): if os.path.isdir(git_mirror_metadatadir):
common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
if os.path.isdir(git_mirror_statsdir): if os.path.isdir(git_mirror_statsdir):
common.local_rsync(options, git_mirror_statsdir + '/', 'stats/') common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
ssh_private_key_file = _ssh_key_from_debug_keystore() ssh_private_key_file = _ssh_key_from_debug_keystore()
# this is needed for GitPython to find the SSH key # this is needed for GitPython to find the SSH key
@ -484,9 +484,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
cwd=repo_basedir, cwd=repo_basedir,
) )
common.local_rsync( common.local_rsync(
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/' options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
) )
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/') common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True) mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata") mirror_git_repo.index.commit("update app metadata")

123
tests/deploy.TestCase
View File

@ -178,7 +178,8 @@ class DeployTest(unittest.TestCase):
fake_apk = repo / 'fake.apk' fake_apk = repo / 'fake.apk'
with fake_apk.open('w') as fp: with fake_apk.open('w') as fp:
fp.write('not an APK, but has the right filename') fp.write('not an APK, but has the right filename')
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] for i in fdroidserver.deploy.INDEX_FILES:
fake_index = repo / i
with fake_index.open('w') as fp: with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename') fp.write('not an index, but has the right filename')
url = Path('url') url = Path('url')
@ -328,32 +329,17 @@ class DeployTest(unittest.TestCase):
'--delete-after', '--delete-after',
'--safe-links', '--safe-links',
'--quiet', '--quiet',
'--include',
"*/",
'--include',
'repo/entry.jar', 'repo/entry.jar',
'--include',
'repo/entry.json', 'repo/entry.json',
'--include',
'repo/entry.json.asc', 'repo/entry.json.asc',
'--include',
'repo/index-v1.jar', 'repo/index-v1.jar',
'--include',
'repo/index-v1.json', 'repo/index-v1.json',
'--include',
'repo/index-v1.json.asc', 'repo/index-v1.json.asc',
'--include',
'repo/index-v2.json', 'repo/index-v2.json',
'--include',
'repo/index-v2.json.asc', 'repo/index-v2.json.asc',
'--include',
'repo/index.jar', 'repo/index.jar',
'--include',
'repo/index.xml', 'repo/index.xml',
'--exclude', 'example.com:/var/www/fdroid/repo/',
'*',
'repo',
'example.com:/var/www/fdroid',
], ],
) )
elif call_iteration == 1: elif call_iteration == 1:
@ -384,6 +370,34 @@ class DeployTest(unittest.TestCase):
'example.com:/var/www/fdroid', 'example.com:/var/www/fdroid',
], ],
) )
# elif call_iteration == 1:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--quiet',
# 'repo',
# serverwebroot,
# ],
# )
# elif call_iteration == 2:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--quiet',
# 'Sym.apk',
# 'Sym.apk.asc',
# 'Sym.apk.sig',
# 'example.com:/var/www/fdroid',
# ],
# )
else: else:
self.fail('unexpected subprocess.call invocation') self.fail('unexpected subprocess.call invocation')
call_iteration += 1 call_iteration += 1
@ -508,32 +522,17 @@ class DeployTest(unittest.TestCase):
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.deploy.config['identity_file'],
'--include',
"*/",
'--include',
'archive/entry.jar', 'archive/entry.jar',
'--include',
'archive/entry.json', 'archive/entry.json',
'--include',
'archive/entry.json.asc', 'archive/entry.json.asc',
'--include',
'archive/index-v1.jar', 'archive/index-v1.jar',
'--include',
'archive/index-v1.json', 'archive/index-v1.json',
'--include',
'archive/index-v1.json.asc', 'archive/index-v1.json.asc',
'--include',
'archive/index-v2.json', 'archive/index-v2.json',
'--include',
'archive/index-v2.json.asc', 'archive/index-v2.json.asc',
'--include',
'archive/index.jar', 'archive/index.jar',
'--include',
'archive/index.xml', 'archive/index.xml',
'--exclude', "example.com:/var/www/fdroid/archive/",
"*",
'archive',
url,
], ],
) )
elif call_iteration == 1: elif call_iteration == 1:
@ -548,10 +547,25 @@ class DeployTest(unittest.TestCase):
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.deploy.config['identity_file'],
'archive', "example.com:/var/www/fdroid/archive/",
url,
], ],
) )
# elif call_iteration == 1:
# self.assertListEqual(
# cmd,
# [
# 'rsync',
# '--archive',
# '--delete-after',
# '--safe-links',
# '--verbose',
# '-e',
# 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
# + fdroidserver.deploy.config['identity_file'],
# 'archive',
# serverwebroot,
# ],
# )
else: else:
self.fail('unexpected subprocess.call invocation') self.fail('unexpected subprocess.call invocation')
call_iteration += 1 call_iteration += 1
@ -881,14 +895,20 @@ class DeployTest(unittest.TestCase):
from libcloud.storage.base import Container from libcloud.storage.base import Container
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["awsbucket"] = "bucket" fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey" fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd" fdroidserver.deploy.config["s3cmd"] = "s3cmd"
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
repo_section = 'repo' repo_section = 'repo'
os.chdir(self.testdir) os.chdir(self.testdir)
@ -937,14 +957,20 @@ class DeployTest(unittest.TestCase):
from libcloud.storage.base import Container from libcloud.storage.base import Container
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["awsbucket"] = "bucket" fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey" fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd" fdroidserver.deploy.config["s3cmd"] = "s3cmd"
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = True
repo_section = 'repo' repo_section = 'repo'
os.chdir(self.testdir) os.chdir(self.testdir)
@ -1102,31 +1128,16 @@ class DeployTest(unittest.TestCase):
'--delete', '--delete',
'--chmod=Da+rx,Fa-x,a+r,u+w', '--chmod=Da+rx,Fa-x,a+r,u+w',
'--quiet', '--quiet',
'--include',
"*/",
'--include',
'repo/entry.jar', 'repo/entry.jar',
'--include',
'repo/entry.json', 'repo/entry.json',
'--include',
'repo/entry.json.asc', 'repo/entry.json.asc',
'--include',
'repo/index-v1.jar', 'repo/index-v1.jar',
'--include',
'repo/index-v1.json', 'repo/index-v1.json',
'--include',
'repo/index-v1.json.asc', 'repo/index-v1.json.asc',
'--include',
'repo/index-v2.json', 'repo/index-v2.json',
'--include',
'repo/index-v2.json.asc', 'repo/index-v2.json.asc',
'--include',
'repo/index.jar', 'repo/index.jar',
'--include',
'repo/index.xml', 'repo/index.xml',
'--exclude',
"*",
'repo/',
"git-mirror/fdroid/repo/", "git-mirror/fdroid/repo/",
], ],
) )