mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
Compare commits
9 Commits
8ec3a2a3db
...
d30ab9d12b
Author | SHA1 | Date | |
---|---|---|---|
|
d30ab9d12b | ||
|
11bd65b0f8 | ||
|
604fe1c275 | ||
|
64da41de60 | ||
|
8d8d93b1cb | ||
|
2b1b6c842d | ||
|
bae8858607 | ||
|
4341343fa4 | ||
|
76a31463b3 |
@ -96,6 +96,15 @@ def _get_index_includes(base_dir):
|
|||||||
return index_includes
|
return index_includes
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_missing_files(files: List[str]) -> List[str]:
|
||||||
|
"""Remove files that are missing from the file system."""
|
||||||
|
existing = []
|
||||||
|
for f in files:
|
||||||
|
if os.path.exists(f):
|
||||||
|
existing.append(f)
|
||||||
|
return existing
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
|
def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
|
||||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||||
|
|
||||||
@ -299,8 +308,9 @@ def update_remote_storage_with_rclone(
|
|||||||
|
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
sources = _get_index_file_paths(repo_section)
|
sources = _get_index_file_paths(repo_section)
|
||||||
|
sources = _remove_missing_files(sources)
|
||||||
else:
|
else:
|
||||||
sources = repo_section
|
sources = [repo_section]
|
||||||
|
|
||||||
for source in sources:
|
for source in sources:
|
||||||
if isinstance(config['rclone_config'], str):
|
if isinstance(config['rclone_config'], str):
|
||||||
@ -429,6 +439,7 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
|
|||||||
for name in files
|
for name in files
|
||||||
]
|
]
|
||||||
files_to_upload = list(set(files_to_upload) & set(index_files))
|
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||||
|
files_to_upload = _remove_missing_files(files_to_upload)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
files_to_upload = [
|
files_to_upload = [
|
||||||
@ -534,7 +545,10 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||||||
is_index_only = serverwebroot.get('index_only', False)
|
is_index_only = serverwebroot.get('index_only', False)
|
||||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
rsyncargs += _get_index_file_paths(repo_section)
|
files_to_upload = _get_index_file_paths(repo_section)
|
||||||
|
files_to_upload = _remove_missing_files(files_to_upload)
|
||||||
|
|
||||||
|
rsyncargs += files_to_upload
|
||||||
rsyncargs += [f'{url}/{repo_section}/']
|
rsyncargs += [f'{url}/{repo_section}/']
|
||||||
logging.info(rsyncargs)
|
logging.info(rsyncargs)
|
||||||
if subprocess.call(rsyncargs) != 0:
|
if subprocess.call(rsyncargs) != 0:
|
||||||
@ -732,10 +746,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||||
# make sure both paths have exactly one trailing slash
|
# make sure both paths have exactly one trailing slash
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
files_to_sync = [
|
files_to_sync = _get_index_file_paths(str(workspace_dir / repo_section))
|
||||||
str(workspace_dir / repo_section / index_file)
|
files_to_sync = _remove_missing_files(files_to_sync)
|
||||||
for index_file in INDEX_FILES
|
|
||||||
]
|
|
||||||
else:
|
else:
|
||||||
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
|
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
|
||||||
common.local_rsync(
|
common.local_rsync(
|
||||||
@ -784,9 +796,11 @@ def upload_to_servergitmirror(
|
|||||||
logging.info('Mirroring to: ' + remote_url)
|
logging.info('Mirroring to: ' + remote_url)
|
||||||
|
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
local_repo.index.add(
|
files_to_upload = _get_index_file_paths(
|
||||||
_get_index_file_paths(os.path.join('fdroid', repo_section))
|
os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
|
||||||
)
|
)
|
||||||
|
files_to_upload = _remove_missing_files(files_to_upload)
|
||||||
|
local_repo.index.add(files_to_upload)
|
||||||
else:
|
else:
|
||||||
# sadly index.add don't allow the --all parameter
|
# sadly index.add don't allow the --all parameter
|
||||||
logging.debug('Adding all files to git mirror')
|
logging.debug('Adding all files to git mirror')
|
||||||
@ -1075,8 +1089,6 @@ def push_binary_transparency(git_repo_path, git_remote):
|
|||||||
drive.
|
drive.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import git
|
|
||||||
|
|
||||||
logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote))
|
logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote))
|
||||||
|
|
||||||
if os.path.isdir(os.path.dirname(git_remote)):
|
if os.path.isdir(os.path.dirname(git_remote)):
|
||||||
|
@ -136,7 +136,7 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.common.options = Options
|
fdroidserver.common.options = Options
|
||||||
|
|
||||||
# write out destination path
|
# write out destination path
|
||||||
destination = Path('some_bucket_folder/fdroid')
|
destination = Path('test_bucket_folder/fdroid')
|
||||||
destination.mkdir(parents=True, exist_ok=True)
|
destination.mkdir(parents=True, exist_ok=True)
|
||||||
dest_apk = Path(destination) / fake_apk
|
dest_apk = Path(destination) / fake_apk
|
||||||
dest_index = Path(destination) / fake_index
|
dest_index = Path(destination) / fake_index
|
||||||
@ -180,7 +180,7 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.common.options = Options
|
fdroidserver.common.options = Options
|
||||||
|
|
||||||
# write out destination path
|
# write out destination path
|
||||||
destination = Path('some_bucket_folder/fdroid')
|
destination = Path('test_bucket_folder/fdroid')
|
||||||
destination.mkdir(parents=True, exist_ok=True)
|
destination.mkdir(parents=True, exist_ok=True)
|
||||||
dest_apk = Path(destination) / fake_apk
|
dest_apk = Path(destination) / fake_apk
|
||||||
dest_index = Path(destination) / fake_index
|
dest_index = Path(destination) / fake_index
|
||||||
@ -429,10 +429,17 @@ class DeployTest(unittest.TestCase):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||||
os.mkdir('repo')
|
os.mkdir(repo_section)
|
||||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||||
|
|
||||||
|
fake_files = fdroidserver.deploy.INDEX_FILES
|
||||||
|
for filename in fake_files:
|
||||||
|
fake_file = Path(repo_section) / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||||
fdroidserver.deploy.update_serverwebroot(
|
fdroidserver.deploy.update_serverwebroot(
|
||||||
{'url': url, 'index_only': True}, repo_section
|
{'url': url, 'index_only': True}, repo_section
|
||||||
@ -583,10 +590,18 @@ class DeployTest(unittest.TestCase):
|
|||||||
call_iteration += 1
|
call_iteration += 1
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||||
fdroidserver.deploy.update_serverwebroot(
|
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||||
{'url': url, 'index_only': True}, repo_section
|
os.mkdir(repo_section)
|
||||||
)
|
fake_files = fdroidserver.deploy.INDEX_FILES
|
||||||
|
for filename in fake_files:
|
||||||
|
fake_file = Path(repo_section) / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
|
fdroidserver.deploy.update_serverwebroot(
|
||||||
|
{'url': url, 'index_only': True}, repo_section
|
||||||
|
)
|
||||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||||
|
|
||||||
@unittest.skipIf(
|
@unittest.skipIf(
|
||||||
|
Loading…
Reference in New Issue
Block a user