refactor: use _get_index_file_paths() instead of this mix of --include/--exclude
This commit is contained in:
proletarius101
parent
7064160fbf
commit
568851d11b
|
|
@ -3997,7 +3997,7 @@ def get_app_display_name(app):
|
|||
return app.get('AutoName') or app['id']
|
||||
|
||||
|
||||
def local_rsync(options, fromdir, todir, args: List[str] = []):
|
||||
def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
|
||||
"""Rsync method for local to local copying of things.
|
||||
|
||||
This is an rsync wrapper with all the settings for safe use within
|
||||
|
|
@ -4015,8 +4015,8 @@ def local_rsync(options, fromdir, todir, args: List[str] = []):
|
|||
if options.quiet:
|
||||
rsyncargs += ['--quiet']
|
||||
rsyncargs += args
|
||||
logging.debug(' '.join(rsyncargs + [fromdir, todir]))
|
||||
if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
|
||||
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
|
||||
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -333,14 +333,16 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||
url = serverwebroot['url']
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||
if options.index_only:
|
||||
rsyncargs += ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*'] + [repo_section, url]
|
||||
rsyncargs += _get_index_file_paths(repo_section)
|
||||
rsyncargs += [f'{url}/{repo_section}/']
|
||||
logging.info(rsyncargs)
|
||||
if subprocess.call(rsyncargs) != 0:
|
||||
raise FDroidException()
|
||||
else:
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
|
||||
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
# upload "current version" symlinks if requested
|
||||
if config['make_current_version_link'] and repo_section == 'repo':
|
||||
|
|
@ -397,7 +399,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
|
|||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
common.local_rsync(options,
|
||||
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
|
||||
[os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
|
||||
repo_section.rstrip('/') + '/')
|
||||
|
||||
offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
|
||||
|
|
@ -418,7 +420,7 @@ def update_localcopy(repo_section, local_copy_dir):
|
|||
raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
|
||||
|
||||
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
||||
common.local_rsync(options, repo_section, local_copy_dir)
|
||||
common.local_rsync(options, [repo_section], local_copy_dir)
|
||||
|
||||
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
|
||||
if os.path.isdir(os.path.join(offline_copy, '.git')):
|
||||
|
|
@ -479,18 +481,15 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
shutil.rmtree(archive_path, ignore_errors=True)
|
||||
|
||||
if options.index_only:
|
||||
rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
|
||||
|
||||
# rsync is very particular about trailing slashes
|
||||
common.local_rsync(options,
|
||||
repo_section.rstrip('/') + '/',
|
||||
git_repodir.rstrip('/') + '/',
|
||||
args=rsyncargs)
|
||||
_get_index_file_paths(repo_section),
|
||||
git_repodir.rstrip('/') + '/')
|
||||
else:
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
common.local_rsync(options,
|
||||
repo_section.rstrip('/') + '/',
|
||||
[repo_section.rstrip('/') + '/'],
|
||||
git_repodir.rstrip('/') + '/')
|
||||
|
||||
# use custom SSH command if identity_file specified
|
||||
|
|
|
|||
|
|
@ -321,11 +321,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
|
||||
os.chdir(repo_basedir)
|
||||
if os.path.isdir(git_mirror_repodir):
|
||||
common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
|
||||
common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
|
||||
if os.path.isdir(git_mirror_metadatadir):
|
||||
common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
|
||||
common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
|
||||
if os.path.isdir(git_mirror_statsdir):
|
||||
common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
|
||||
common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
|
||||
|
||||
ssh_private_key_file = _ssh_key_from_debug_keystore()
|
||||
# this is needed for GitPython to find the SSH key
|
||||
|
|
@ -427,9 +427,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
cwd=repo_basedir,
|
||||
)
|
||||
common.local_rsync(
|
||||
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
|
||||
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
|
||||
)
|
||||
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
|
||||
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
|
||||
mirror_git_repo.git.add(all=True)
|
||||
mirror_git_repo.index.commit("update app metadata")
|
||||
|
||||
|
|
|
|||
|
|
@ -128,9 +128,10 @@ class DeployTest(unittest.TestCase):
|
|||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
for i in fdroidserver.deploy.INDEX_FILES:
|
||||
fake_index = repo / i
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
url = Path('url')
|
||||
url.mkdir()
|
||||
|
||||
|
|
@ -275,32 +276,17 @@ class DeployTest(unittest.TestCase):
|
|||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--quiet',
|
||||
'--include',
|
||||
"*/",
|
||||
'--include',
|
||||
'repo/entry.jar',
|
||||
'--include',
|
||||
'repo/entry.json',
|
||||
'--include',
|
||||
'repo/entry.json.asc',
|
||||
'--include',
|
||||
'repo/index-v1.jar',
|
||||
'--include',
|
||||
'repo/index-v1.json',
|
||||
'--include',
|
||||
'repo/index-v1.json.asc',
|
||||
'--include',
|
||||
'repo/index-v2.json',
|
||||
'--include',
|
||||
'repo/index-v2.json.asc',
|
||||
'--include',
|
||||
'repo/index.jar',
|
||||
'--include',
|
||||
'repo/index.xml',
|
||||
'--exclude',
|
||||
'*',
|
||||
'repo',
|
||||
'example.com:/var/www/fdroid',
|
||||
'example.com:/var/www/fdroid/repo/',
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
|
|
@ -331,6 +317,34 @@ class DeployTest(unittest.TestCase):
|
|||
'example.com:/var/www/fdroid',
|
||||
],
|
||||
)
|
||||
# elif call_iteration == 1:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--quiet',
|
||||
# 'repo',
|
||||
# serverwebroot,
|
||||
# ],
|
||||
# )
|
||||
# elif call_iteration == 2:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--quiet',
|
||||
# 'Sym.apk',
|
||||
# 'Sym.apk.asc',
|
||||
# 'Sym.apk.sig',
|
||||
# 'example.com:/var/www/fdroid',
|
||||
# ],
|
||||
# )
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
|
|
@ -454,32 +468,17 @@ class DeployTest(unittest.TestCase):
|
|||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
'--include',
|
||||
"*/",
|
||||
'--include',
|
||||
'archive/entry.jar',
|
||||
'--include',
|
||||
'archive/entry.json',
|
||||
'--include',
|
||||
'archive/entry.json.asc',
|
||||
'--include',
|
||||
'archive/index-v1.jar',
|
||||
'--include',
|
||||
'archive/index-v1.json',
|
||||
'--include',
|
||||
'archive/index-v1.json.asc',
|
||||
'--include',
|
||||
'archive/index-v2.json',
|
||||
'--include',
|
||||
'archive/index-v2.json.asc',
|
||||
'--include',
|
||||
'archive/index.jar',
|
||||
'--include',
|
||||
'archive/index.xml',
|
||||
'--exclude',
|
||||
"*",
|
||||
'archive',
|
||||
url,
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
|
|
@ -494,10 +493,25 @@ class DeployTest(unittest.TestCase):
|
|||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
'archive',
|
||||
url,
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
# elif call_iteration == 1:
|
||||
# self.assertListEqual(
|
||||
# cmd,
|
||||
# [
|
||||
# 'rsync',
|
||||
# '--archive',
|
||||
# '--delete-after',
|
||||
# '--safe-links',
|
||||
# '--verbose',
|
||||
# '-e',
|
||||
# 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
# + fdroidserver.deploy.config['identity_file'],
|
||||
# 'archive',
|
||||
# serverwebroot,
|
||||
# ],
|
||||
# )
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
|
|
@ -827,14 +841,20 @@ class DeployTest(unittest.TestCase):
|
|||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
|
|
@ -883,14 +903,20 @@ class DeployTest(unittest.TestCase):
|
|||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = True
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
|
|
@ -1048,31 +1074,16 @@ class DeployTest(unittest.TestCase):
|
|||
'--delete',
|
||||
'--chmod=Da+rx,Fa-x,a+r,u+w',
|
||||
'--quiet',
|
||||
'--include',
|
||||
"*/",
|
||||
'--include',
|
||||
'repo/entry.jar',
|
||||
'--include',
|
||||
'repo/entry.json',
|
||||
'--include',
|
||||
'repo/entry.json.asc',
|
||||
'--include',
|
||||
'repo/index-v1.jar',
|
||||
'--include',
|
||||
'repo/index-v1.json',
|
||||
'--include',
|
||||
'repo/index-v1.json.asc',
|
||||
'--include',
|
||||
'repo/index-v2.json',
|
||||
'--include',
|
||||
'repo/index-v2.json.asc',
|
||||
'--include',
|
||||
'repo/index.jar',
|
||||
'--include',
|
||||
'repo/index.xml',
|
||||
'--exclude',
|
||||
"*",
|
||||
'repo/',
|
||||
"git-mirror/fdroid/repo/",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue