1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00

Compare commits

...

2 Commits

Author SHA1 Message Date
proletarius101
50a38fdea4 Merge branch 'index-only-deploy-mode' into 'master'
Add index only deployment mode

Closes #1181

See merge request fdroid/fdroidserver!1420
2024-06-04 13:58:48 +00:00
proletarius101
be0f075914
fix(deploy): fix errors introduced by merging 2024-06-04 21:58:29 +08:00
2 changed files with 165 additions and 123 deletions

View File

@ -25,10 +25,10 @@ import re
import subprocess import subprocess
import time import time
import urllib import urllib
from typing import Dict, Optional, List from typing import Dict, List
from git import Repo from git import Repo
import yaml import yaml
from argparse import ArgumentParser, Namespace from argparse import ArgumentParser
import logging import logging
from shlex import split from shlex import split
import shutil import shutil
@ -41,7 +41,6 @@ from . import index
from .exception import FDroidException from .exception import FDroidException
config = None config = None
options: Optional[Namespace] = None
start_timestamp = time.gmtime() start_timestamp = time.gmtime()
GIT_BRANCH = 'master' GIT_BRANCH = 'master'
@ -126,7 +125,9 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
if config['s3cmd'] is True and config['rclone'] is not True: if config['s3cmd'] is True and config['rclone'] is not True:
update_awsbucket_s3cmd(repo_section, is_index_only) update_awsbucket_s3cmd(repo_section, is_index_only)
if config['rclone'] is True and config['s3cmd'] is not True: if config['rclone'] is True and config['s3cmd'] is not True:
update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) update_remote_storage_with_rclone(
repo_section, is_index_only, verbose, quiet
)
elif common.set_command_in_config('s3cmd'): elif common.set_command_in_config('s3cmd'):
update_awsbucket_s3cmd(repo_section, is_index_only) update_awsbucket_s3cmd(repo_section, is_index_only)
@ -191,8 +192,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
) )
if is_index_only: if is_index_only:
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed') logging.debug(
.format(path=repo_section, url=s3url)) _('s3cmd syncs indexes from {path} to {url} and deletes removed').format(
path=repo_section, url=s3url
)
)
sync_indexes_flags = [] sync_indexes_flags = []
sync_indexes_flags.extend(_get_index_includes(repo_section)) sync_indexes_flags.extend(_get_index_includes(repo_section))
sync_indexes_flags.append('--delete-removed') sync_indexes_flags.append('--delete-removed')
@ -201,7 +205,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
sync_indexes_flags.append('--no-check-md5') sync_indexes_flags.append('--no-check-md5')
else: else:
sync_indexes_flags.append('--check-md5') sync_indexes_flags.append('--check-md5')
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url]) returncode = subprocess.call(
s3cmd_sync + sync_indexes_flags + [repo_section, s3url]
)
if returncode != 0: if returncode != 0:
raise FDroidException() raise FDroidException()
else: else:
@ -222,8 +228,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
if returncode != 0: if returncode != 0:
raise FDroidException() raise FDroidException()
logging.debug(_('s3cmd sync indexes {path} to {url} and delete') logging.debug(
.format(path=repo_section, url=s3url)) _('s3cmd sync indexes {path} to {url} and delete').format(
path=repo_section, url=s3url
)
)
s3cmd_sync.append('--delete-removed') s3cmd_sync.append('--delete-removed')
s3cmd_sync.append('--delete-after') s3cmd_sync.append('--delete-after')
if options.no_checksum: if options.no_checksum:
@ -234,7 +243,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
raise FDroidException() raise FDroidException()
def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False): def update_remote_storage_with_rclone(
repo_section, is_index_only=False, verbose=False, quiet=False
):
""" """
Upload fdroid repo folder to remote storage using rclone sync. Upload fdroid repo folder to remote storage using rclone sync.
@ -353,10 +364,7 @@ def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose
) )
logging.debug( logging.debug(
"rclone sync all files in " "rclone sync all files in " + source + ' to ' + complete_remote_path
+ source
+ ' to '
+ complete_remote_path
) )
if subprocess.call(rclone_sync_command) != 0: if subprocess.call(rclone_sync_command) != 0:
@ -412,12 +420,22 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
objs[obj.name] = obj objs[obj.name] = obj
if is_index_only: if is_index_only:
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)] index_files = [
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)
]
files_to_upload = [
os.path.join(root, name)
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
for name in files
]
files_to_upload = list(set(files_to_upload) & set(index_files)) files_to_upload = list(set(files_to_upload) & set(index_files))
else: else:
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] files_to_upload = [
os.path.join(root, name)
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
for name in files
]
for file_to_upload in files_to_upload: for file_to_upload in files_to_upload:
upload = False upload = False
@ -528,9 +546,15 @@ def update_serverwebroot(serverwebroot, repo_section):
if subprocess.call(rsyncargs + [repo_section, url]) != 0: if subprocess.call(rsyncargs + [repo_section, url]) != 0:
raise FDroidException() raise FDroidException()
# upload "current version" symlinks if requested # upload "current version" symlinks if requested
if config and config.get('make_current_version_link') and repo_section == 'repo': if (
config
and config.get('make_current_version_link')
and repo_section == 'repo'
):
links_to_upload = [] links_to_upload = []
for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): for f in (
glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig')
):
if os.path.islink(f): if os.path.islink(f):
links_to_upload.append(f) links_to_upload.append(f)
if len(links_to_upload) > 0: if len(links_to_upload) > 0:
@ -708,35 +732,42 @@ def update_servergitmirrors(servergitmirrors, repo_section):
# trailing slashes have a meaning in rsync which is not needed here, so # trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash # make sure both paths have exactly one trailing slash
if is_index_only: if is_index_only:
files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES] files_to_sync = [
str(workspace_dir / repo_section / index_file)
for index_file in INDEX_FILES
]
else: else:
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/'] files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
common.local_rsync(common.get_options(), common.local_rsync(
files_to_sync, common.get_options(), files_to_sync, git_repodir.rstrip('/') + '/'
git_repodir.rstrip('/') + '/') )
upload_to_servergitmirror(mirror_config=d, upload_to_servergitmirror(
local_repo=repo, mirror_config=d,
enabled_remotes=enabled_remotes, local_repo=repo,
repo_section=repo_section, enabled_remotes=enabled_remotes,
is_index_only=is_index_only, repo_section=repo_section,
fdroid_dir=git_fdroiddir, is_index_only=is_index_only,
git_mirror_path=str(git_mirror_path), fdroid_dir=git_fdroiddir,
ssh_cmd=ssh_cmd, git_mirror_path=str(git_mirror_path),
progress=progress) ssh_cmd=ssh_cmd,
progress=progress,
)
if progress: if progress:
progressbar.done() progressbar.done()
def upload_to_servergitmirror(mirror_config: Dict[str, str], def upload_to_servergitmirror(
local_repo: Repo, mirror_config: Dict[str, str],
enabled_remotes: List[str], local_repo: Repo,
repo_section: str, enabled_remotes: List[str],
is_index_only: bool, repo_section: str,
fdroid_dir: str, is_index_only: bool,
git_mirror_path: str, fdroid_dir: str,
ssh_cmd: str, git_mirror_path: str,
progress: git.RemoteProgress) -> None: ssh_cmd: str,
progress: git.RemoteProgress,
) -> None:
remote_branch_name = GIT_BRANCH remote_branch_name = GIT_BRANCH
local_branch_name = local_repo.active_branch.name local_branch_name = local_repo.active_branch.name
@ -754,7 +785,9 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
logging.info("git status:", local_repo.git.status()) logging.info("git status:", local_repo.git.status())
if is_index_only: if is_index_only:
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section))) local_repo.index.add(
_get_index_file_paths(os.path.join('fdroid', repo_section))
)
else: else:
# sadly index.add don't allow the --all parameter # sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror') logging.debug('Adding all files to git mirror')
@ -763,16 +796,13 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
logging.debug('Committing files into git mirror') logging.debug('Committing files into git mirror')
local_repo.index.commit("fdroidserver git-mirror") local_repo.index.commit("fdroidserver git-mirror")
# only deploy to GitLab Artifacts if too big for GitLab Pages # only deploy to GitLab Artifacts if too big for GitLab Pages
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE: if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
gitlab_ci_job_name = 'pages' gitlab_ci_job_name = 'pages'
else: else:
gitlab_ci_job_name = 'GitLab Artifacts' gitlab_ci_job_name = 'GitLab Artifacts'
logging.warning( logging.warning(
_( _('Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!')
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
)
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
) )
@ -803,19 +833,29 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
logging.info(_('Pushing to {url}').format(url=remote.url)) logging.info(_('Pushing to {url}').format(url=remote.url))
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
pushinfos = remote.push( pushinfos = remote.push(
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress f"{local_branch_name}:{remote_branch_name}",
force=True,
set_upstream=True,
progress=progress,
) )
for pushinfo in pushinfos: for pushinfo in pushinfos:
if pushinfo.flags & (git.remote.PushInfo.ERROR if pushinfo.flags & (
| git.remote.PushInfo.REJECTED git.remote.PushInfo.ERROR
| git.remote.PushInfo.REMOTE_FAILURE | git.remote.PushInfo.REJECTED
| git.remote.PushInfo.REMOTE_REJECTED): | git.remote.PushInfo.REMOTE_FAILURE
| git.remote.PushInfo.REMOTE_REJECTED
):
# Show potentially useful messages from git remote # Show potentially useful messages from git remote
for line in progress.other_lines: for line in progress.other_lines:
if line.startswith('remote:'): if line.startswith('remote:'):
logging.debug(line) logging.debug(line)
raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags) raise FDroidException(
+ ' ' + pushinfo.summary) remote.url
+ ' push failed: '
+ str(pushinfo.flags)
+ ' '
+ pushinfo.summary
)
else: else:
logging.debug(remote.url + ': ' + pushinfo.summary) logging.debug(remote.url + ': ' + pushinfo.summary)
@ -1083,14 +1123,6 @@ def main():
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("-i", "--identity-file", default=None,
help=_("Specify an identity file to provide to SSH for rsyncing"))
parser.add_argument("--local-copy-dir", default=None,
help=_("Specify a local folder to sync the repo to"))
parser.add_argument("--no-checksum", action="store_true", default=False,
help=_("Don't use rsync checksums"))
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
help=_("If a git mirror gets to big, allow the archive to be deleted"))
parser.add_argument( parser.add_argument(
"-i", "-i",
"--identity-file", "--identity-file",
@ -1114,8 +1146,8 @@ def main():
default=False, default=False,
help=_("If a git mirror gets to big, allow the archive to be deleted"), help=_("If a git mirror gets to big, allow the archive to be deleted"),
) )
options = parser.parse_args() options = common.parse_args(parser)
config = common.read_config(options) config = common.read_config()
if config.get('nonstandardwebroot') is True: if config.get('nonstandardwebroot') is True:
standardwebroot = False standardwebroot = False

View File

@ -40,7 +40,7 @@ class DeployTest(unittest.TestCase):
self._td = mkdtemp() self._td = mkdtemp()
self.testdir = self._td.name self.testdir = self._td.name
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.config = {} fdroidserver.deploy.config = {}
fdroidserver.deploy.USER_RCLONE_CONF = False fdroidserver.deploy.USER_RCLONE_CONF = False
@ -73,7 +73,7 @@ class DeployTest(unittest.TestCase):
url1.mkdir(parents=True) url1.mkdir(parents=True)
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk0 = url0 / fake_apk dest_apk0 = url0 / fake_apk
@ -117,7 +117,6 @@ class DeployTest(unittest.TestCase):
with fake_index.open('w') as fp: with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename') fp.write('not an index, but has the right filename')
# write out rclone config for test use # write out rclone config for test use
rclone_config = configparser.ConfigParser() rclone_config = configparser.ConfigParser()
rclone_config.add_section("test-local-config") rclone_config.add_section("test-local-config")
@ -134,7 +133,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.config['rclone'] = True fdroidserver.deploy.config['rclone'] = True
fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
fdroidserver.deploy.options = Options fdroidserver.common.options = Options
# write out destination path # write out destination path
destination = Path('some_bucket_folder/fdroid') destination = Path('some_bucket_folder/fdroid')
@ -178,7 +177,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.config['rclone'] = True fdroidserver.deploy.config['rclone'] = True
fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
fdroidserver.deploy.options = Options fdroidserver.common.options = Options
# write out destination path # write out destination path
destination = Path('some_bucket_folder/fdroid') destination = Path('some_bucket_folder/fdroid')
@ -210,9 +209,9 @@ class DeployTest(unittest.TestCase):
url.mkdir() url.mkdir()
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk dest_apk = Path(url) / fake_apk
@ -238,8 +237,8 @@ class DeployTest(unittest.TestCase):
url.mkdir() url.mkdir()
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk dest_apk = Path(url) / fake_apk
@ -269,7 +268,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.common.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = True fdroidserver.common.options.quiet = True
fdroidserver.deploy.options.index_only = False fdroidserver.common.options.index_only = False
fdroidserver.deploy.config = {'make_current_version_link': True} fdroidserver.deploy.config = {'make_current_version_link': True}
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'repo' repo_section = 'repo'
@ -358,12 +357,12 @@ class DeployTest(unittest.TestCase):
def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self): def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.common.options.no_checksum = True
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = True fdroidserver.deploy.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'repo' repo_section = 'repo'
@ -382,6 +381,7 @@ class DeployTest(unittest.TestCase):
'--delete-after', '--delete-after',
'--safe-links', '--safe-links',
'--quiet', '--quiet',
'repo/altstore-index.json',
'repo/entry.jar', 'repo/entry.jar',
'repo/entry.json', 'repo/entry.json',
'repo/entry.json.asc', 'repo/entry.json.asc',
@ -447,7 +447,7 @@ class DeployTest(unittest.TestCase):
fdroidserver.common.options.verbose = True fdroidserver.common.options.verbose = True
fdroidserver.common.options.quiet = False fdroidserver.common.options.quiet = False
fdroidserver.common.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.index_only = False fdroidserver.common.options.index_only = False
fdroidserver.deploy.config = {'identity_file': './id_rsa'} fdroidserver.deploy.config = {'identity_file': './id_rsa'}
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
repo_section = 'archive' repo_section = 'archive'
@ -522,11 +522,11 @@ class DeployTest(unittest.TestCase):
def test_update_serverwebroot_with_id_file_in_index_only_mode(self): def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_chcksum = False fdroidserver.common.options.no_chcksum = False
fdroidserver.deploy.options.verbose = True fdroidserver.common.options.verbose = True
fdroidserver.deploy.options.quiet = False fdroidserver.common.options.quiet = False
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['identity_file'] = './id_rsa'
fdroidserver.deploy.config['make_current_version_link'] = False fdroidserver.deploy.config['make_current_version_link'] = False
url = "example.com:/var/www/fdroid" url = "example.com:/var/www/fdroid"
@ -549,6 +549,7 @@ class DeployTest(unittest.TestCase):
'-e', '-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'], + fdroidserver.deploy.config['identity_file'],
'archive/altstore-index.json',
'archive/entry.jar', 'archive/entry.jar',
'archive/entry.json', 'archive/entry.json',
'archive/entry.json.asc', 'archive/entry.json.asc',
@ -592,8 +593,8 @@ class DeployTest(unittest.TestCase):
not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set' not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
) )
def test_upload_to_virustotal(self): def test_upload_to_virustotal(self):
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.verbose = True fdroidserver.common.options.verbose = True
virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey) fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
@ -611,10 +612,10 @@ class DeployTest(unittest.TestCase):
def test_update_awsbucket_s3cmd(self): def test_update_awsbucket_s3cmd(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.common.options.no_checksum = True
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -651,6 +652,8 @@ class DeployTest(unittest.TestCase):
'--acl-public', '--acl-public',
'--quiet', '--quiet',
'--exclude', '--exclude',
'repo/altstore-index.json',
'--exclude',
'repo/entry.jar', 'repo/entry.jar',
'--exclude', '--exclude',
'repo/entry.json', 'repo/entry.json',
@ -686,6 +689,8 @@ class DeployTest(unittest.TestCase):
'--acl-public', '--acl-public',
'--quiet', '--quiet',
'--exclude', '--exclude',
'repo/altstore-index.json',
'--exclude',
'repo/entry.jar', 'repo/entry.jar',
'--exclude', '--exclude',
'repo/entry.json', 'repo/entry.json',
@ -742,10 +747,10 @@ class DeployTest(unittest.TestCase):
def test_update_awsbucket_s3cmd_in_index_only_mode(self): def test_update_awsbucket_s3cmd_in_index_only_mode(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.common.options.no_checksum = True
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -782,6 +787,8 @@ class DeployTest(unittest.TestCase):
'--acl-public', '--acl-public',
'--quiet', '--quiet',
'--include', '--include',
'repo/altstore-index.json',
'--include',
'repo/entry.jar', 'repo/entry.jar',
'--include', '--include',
'repo/entry.json', 'repo/entry.json',
@ -820,7 +827,7 @@ class DeployTest(unittest.TestCase):
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
fdroidserver.deploy.update_awsbucket_s3cmd( fdroidserver.deploy.update_awsbucket_s3cmd(
repo_section, index_only=True repo_section, is_index_only=True
) )
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
@ -828,10 +835,10 @@ class DeployTest(unittest.TestCase):
from libcloud.storage.base import Container from libcloud.storage.base import Container
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.common.options.no_checksum = True
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -872,7 +879,10 @@ class DeployTest(unittest.TestCase):
container_name=fdroidserver.deploy.config["awsbucket"] container_name=fdroidserver.deploy.config["awsbucket"]
) )
mock_container.list_objects.assert_called_once_with() mock_container.list_objects.assert_called_once_with()
files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar'] files_to_upload = [
'fdroid/repo/Sym.apk',
f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}",
]
calls = [ calls = [
mock.call( mock.call(
iterator=mock.ANY, iterator=mock.ANY,
@ -889,10 +899,10 @@ class DeployTest(unittest.TestCase):
from libcloud.storage.base import Container from libcloud.storage.base import Container
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True fdroidserver.common.options.no_checksum = True
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -927,7 +937,7 @@ class DeployTest(unittest.TestCase):
mock_driver.get_container.return_value = mock_container mock_driver.get_container.return_value = mock_container
mock_driver.upload_object_via_stream.return_value = None mock_driver.upload_object_via_stream.return_value = None
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True) fdroidserver.deploy.update_awsbucket_libcloud(repo_section, is_index_only=True)
mock_driver.get_container.assert_called_once_with( mock_driver.get_container.assert_called_once_with(
container_name=fdroidserver.deploy.config["awsbucket"] container_name=fdroidserver.deploy.config["awsbucket"]
@ -951,11 +961,11 @@ class DeployTest(unittest.TestCase):
def test_update_servergitmirrors(self): def test_update_servergitmirrors(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.common.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -1002,11 +1012,11 @@ class DeployTest(unittest.TestCase):
def test_update_servergitmirrors_in_index_only_mode(self): def test_update_servergitmirrors_in_index_only_mode(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.common.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)
@ -1062,12 +1072,12 @@ class DeployTest(unittest.TestCase):
def test_upload_to_servergitmirror_in_index_only_mode(self): def test_upload_to_servergitmirror_in_index_only_mode(self):
# setup parameters for this test run # setup parameters for this test run
fdroidserver.deploy.options = mock.Mock() fdroidserver.common.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.common.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False fdroidserver.common.options.verbose = False
fdroidserver.deploy.options.quiet = True fdroidserver.common.options.quiet = True
fdroidserver.deploy.options.identity_file = None fdroidserver.common.options.identity_file = None
config = {} config = {}
fdroidserver.common.fill_config_defaults(config) fdroidserver.common.fill_config_defaults(config)