mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
feat(deploy): add index-only mode to the rclone deployment method
This commit is contained in:
parent
4c0e096a99
commit
cb18d10197
@ -126,12 +126,12 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
|
||||
if config['s3cmd'] is True and config['rclone'] is not True:
|
||||
update_awsbucket_s3cmd(repo_section, is_index_only)
|
||||
if config['rclone'] is True and config['s3cmd'] is not True:
|
||||
update_remote_storage_with_rclone(repo_section, verbose, quiet)
|
||||
update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
|
||||
|
||||
elif common.set_command_in_config('s3cmd'):
|
||||
update_awsbucket_s3cmd(repo_section, is_index_only)
|
||||
elif common.set_command_in_config('rclone'):
|
||||
update_remote_storage_with_rclone(repo_section, verbose, quiet)
|
||||
update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
|
||||
else:
|
||||
update_awsbucket_libcloud(repo_section, is_index_only)
|
||||
|
||||
@ -234,7 +234,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False):
|
||||
"""
|
||||
Upload fdroid repo folder to remote storage using rclone sync.
|
||||
|
||||
@ -286,10 +286,16 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
_('To use rclone, rclone_config and awsbucket must be set in config.yml!')
|
||||
)
|
||||
|
||||
if is_index_only:
|
||||
sources = _get_index_file_paths(repo_section)
|
||||
else:
|
||||
sources = repo_section
|
||||
|
||||
for source in sources:
|
||||
if isinstance(config['rclone_config'], str):
|
||||
rclone_sync_command = (
|
||||
'rclone sync '
|
||||
+ repo_section
|
||||
+ source
|
||||
+ ' '
|
||||
+ config['rclone_config']
|
||||
+ ':'
|
||||
@ -313,7 +319,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
)
|
||||
|
||||
logging.debug(
|
||||
"rclone sync all files in " + repo_section + ' to ' + complete_remote_path
|
||||
"rclone sync all files in " + source + ' to ' + complete_remote_path
|
||||
)
|
||||
|
||||
if subprocess.call(rclone_sync_command) != 0:
|
||||
@ -323,7 +329,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
for remote_config in config['rclone_config']:
|
||||
rclone_sync_command = (
|
||||
'rclone sync '
|
||||
+ repo_section
|
||||
+ source
|
||||
+ ' '
|
||||
+ remote_config
|
||||
+ ':'
|
||||
@ -348,7 +354,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
|
||||
|
||||
logging.debug(
|
||||
"rclone sync all files in "
|
||||
+ repo_section
|
||||
+ source
|
||||
+ ' to '
|
||||
+ complete_remote_path
|
||||
)
|
||||
|
@ -113,6 +113,10 @@ class DeployTest(unittest.TestCase):
|
||||
fake_apk = repo / 'another_fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
|
||||
# write out rclone config for test use
|
||||
rclone_config = configparser.ConfigParser()
|
||||
@ -135,12 +139,61 @@ class DeployTest(unittest.TestCase):
|
||||
# write out destination path
|
||||
destination = Path('some_bucket_folder/fdroid')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
dest_path = Path(destination) / fake_apk
|
||||
self.assertFalse(dest_path.is_file())
|
||||
dest_apk = Path(destination) / fake_apk
|
||||
dest_index = Path(destination) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
repo_section = str(repo)
|
||||
# fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
|
||||
self.assertFalse(dest_path.is_file())
|
||||
self.assertTrue(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
@unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
|
||||
def test_update_remote_storage_with_rclone_in_index_only_mode(self):
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fake_apk = repo / 'another_fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
# write out rclone config for test use
|
||||
rclone_config = configparser.ConfigParser()
|
||||
rclone_config.add_section("test-local-config")
|
||||
rclone_config.set("test-local-config", "type", "local")
|
||||
|
||||
rclone_config_path = Path('rclone_config_path')
|
||||
rclone_config_path.mkdir(parents=True, exist_ok=True)
|
||||
rclone_file = rclone_config_path / 'rclone.conf'
|
||||
with open(rclone_file, 'w') as configfile:
|
||||
rclone_config.write(configfile)
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['rclone'] = True
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.deploy.options = Options
|
||||
|
||||
# write out destination path
|
||||
destination = Path('some_bucket_folder/fdroid')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
dest_apk = Path(destination) / fake_apk
|
||||
dest_index = Path(destination) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
repo_section = str(repo)
|
||||
# fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(
|
||||
repo_section, is_index_only=True
|
||||
)
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
def test_update_serverwebroot(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
@ -178,8 +231,7 @@ class DeployTest(unittest.TestCase):
|
||||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
for i in fdroidserver.deploy.INDEX_FILES:
|
||||
fake_index = repo / i
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
url = Path('url')
|
||||
|
Loading…
Reference in New Issue
Block a user