1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00

Compare commits

...

67 Commits

Author SHA1 Message Date
proletarius101
da5ff1c928
chore(deploy): rename index_only variable to is_index_only for Python idiomaticness 2024-06-04 20:30:35 +08:00
proletarius101
dc3fd0bdd1
chore(deploy): clean up code 2024-06-04 20:27:01 +08:00
proletarius101
c26267a7f9
fix(deploy): remove the --index-only flag 2024-06-04 20:22:56 +08:00
proletarius101
48c48596a8
fix(deploy): $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk doesn't exist when the test is finished 2024-06-04 20:10:13 +08:00
proletarius101
5ff9a19072
style(deploy): rename working_dir_root variable to workspace_dir 2024-06-05 03:52:02 +08:00
proletarius101
78b6e53e42
style(deploy): fix pyflake 2024-06-05 02:38:14 +08:00
proletarius101
40ecaff6fe
fix(deploy): all files are uploaded if the index only mode deployment follows a full mode one
switch the branch before rsyncing
2024-06-03 21:09:03 +08:00
proletarius101
5aeece612f
chore(deploy): convert print to logging for in-order output 2024-06-03 00:58:17 +08:00
proletarius101
abdaced2a5
fix(deploy): all files are uploaded if the index only mode deployment follows a full mode one
create new branches from groud up
2024-06-03 00:51:54 +08:00
proletarius101
3833bc7d6e
fix(deploy): all files are uploaded if the index only mode deployment follows a full mode one
let rsync only sync necessary files
2024-06-03 00:45:45 +08:00
proletarius101
7d123b5578
fix(deploy): all files are uploaded if the index only mode deployment follows a full mode one 2024-06-03 00:30:57 +08:00
proletarius101
59fdaa321c
debug 2024-06-03 00:24:19 +08:00
proletarius101
9441d0e849
fix(deploy): fatal: A branch named 'full' already exists. 2024-06-02 23:37:06 +08:00
proletarius101
a09552207d
test(deploy): make sure the APK file is not uploaded in the index only mode 2024-06-02 23:27:58 +08:00
proletarius101
10c06b8b44
fix(deploy): local files are not synced to the local git repo 2024-06-02 23:06:23 +08:00
proletarius101
d2a867a018
fix(deploy): remove unnecessary additional initial commit
and fix the test error: https://gitlab.com/proletarius101/fdroidserver/-/jobs/6997355250#L6978
2024-06-02 21:13:37 +08:00
proletarius101
a19fc1d607
chore(deploy): list all files in the local git repo 2024-06-02 19:39:05 +08:00
proletarius101
cd2a0fbf29
fix(deploy): files are not pushed to the remote 2024-06-02 19:31:13 +08:00
proletarius101
04c191e4d8
test(deploy): fix fdroidserver.deploy.options is None 2024-05-23 22:40:51 +08:00
proletarius101
5fe1fa9dd3
test(deploy): fix errors 2024-05-20 00:55:28 +08:00
proletarius101
092a8dda2e
fix(deploy): don't recreate the branches 2024-05-20 00:16:27 +08:00
proletarius101
4f09ba2bff
fix: No such file or directory: 'repo/entry.jar' 2024-04-06 22:44:47 +08:00
proletarius101
c148ea3d8e
test 2024-04-06 22:30:50 +08:00
proletarius101
d79ac8e2dc
test 2024-04-06 22:23:36 +08:00
proletarius101
5aef1da1fc
test 2024-04-06 22:19:13 +08:00
proletarius101
e966ac2efe
test 2024-04-06 22:17:40 +08:00
proletarius101
1cc23b20c8
test 2024-04-06 17:46:31 +08:00
proletarius101
a5740f364c
fix(deploy): align the config naming 2024-04-06 17:41:16 +08:00
proletarius101
4f2ec8f2e3
test 2024-04-06 17:20:30 +08:00
proletarius101
5a46b062f4
test 2024-04-06 17:18:23 +08:00
proletarius101
649bb5b1bc
test 2024-04-06 17:16:45 +08:00
proletarius101
462786bb37
test 2024-04-06 17:13:24 +08:00
proletarius101
31f9919073
test 2024-04-06 17:10:12 +08:00
proletarius101
cd23cf659b
fix(deploy): error: The branch 'full' is not fully merged. 2024-04-06 16:49:24 +08:00
proletarius101
d9984dbeea
fix(deploy): error: The following untracked working tree files would be overwritten by checkout 2024-04-06 16:45:03 +08:00
proletarius101
6ddf241713
fix: no files are uploaded 2024-04-06 16:36:38 +08:00
proletarius101
5e0980b98a
fix(deploy): Reference at 'refs/heads/full' does already exist 2024-04-06 16:11:12 +08:00
proletarius101
d2ada6e418
fix: index only mode setting doesn't work if it follows a full mode config 2024-04-06 16:01:51 +08:00
proletarius101
96d1f4190f
fix: index only mode setting doesn't work if it follows a full mode config 2024-04-06 14:53:54 +08:00
Hans-Christoph Steiner
2d267e6cbd fix: avoid making the parent directory redundantly in test case 2024-04-06 05:49:49 +00:00
proletarius101
513b223caa lint: fix pydocstyle 2024-02-28 08:02:35 +00:00
proletarius101
6271de094e fix: ModuleNotFoundError 2024-02-28 08:02:35 +00:00
proletarius101
51b88b784b lint: fix black formatting 2024-02-28 08:02:35 +00:00
proletarius101
7022eabe84 fix(deploy): remove the index-only mode from elsewhere except the config file
To enable per-remote configuration
2024-02-28 08:02:35 +00:00
proletarius101
1e259cb043 style: fix lint 2024-02-28 08:02:35 +00:00
proletarius101
b3c15038da style: fix lint 2024-02-28 08:02:35 +00:00
proletarius101
775316747c style: fix lint 2024-02-28 08:02:35 +00:00
proletarius101
4a793cd67a refactor: use _get_index_file_paths() instead of this mix of --include/--exclude 2024-02-28 08:02:35 +00:00
proletarius101
e05bc9c1db refactor: remove the support of the index only mode when syncing to the local filesystem 2024-02-28 08:02:35 +00:00
proletarius101
6b1c5bae0f chore: revert repository vscode settings 2024-02-28 08:02:35 +00:00
proletarius101
b4b4b20b4c feat(deploy): add index only mode in update_servergitmirrors 2024-02-28 08:02:35 +00:00
proletarius101
d814291577 style(deploy): remove unused imports 2024-02-28 08:02:35 +00:00
proletarius101
707b91e1eb style(deploy): fix pydocstyle errors 2024-02-28 08:02:35 +00:00
proletarius101
34395b0932 test(deploy): add test cases for update_awsbucket_libcloud and fix errors 2024-02-28 08:02:35 +00:00
proletarius101
aa99f27afb test(deploy): add test cases for update_awsbucket_s3cmd 2024-02-28 08:02:35 +00:00
proletarius101
a853d6027e fix(deploy): add test cases for local copy mode and fix issues 2024-02-28 08:02:35 +00:00
proletarius101
738d7dbf4e fix(deploy): add test cases for server webroot mode and fix issues 2024-02-28 08:02:35 +00:00
proletarius101
450b0d9a84 fix(deploy): restore accidentally removed code 2024-02-28 08:02:35 +00:00
proletarius101
7775aff1f2 fix(deploy): fix typo 2024-02-28 08:02:35 +00:00
proletarius101
aa54495388 test(deploy): properly test non-index-only mode 2024-02-28 08:02:35 +00:00
proletarius101
35952418a2 fix(deploy): fix typo 2024-02-28 08:02:35 +00:00
proletarius101
8e6996b25d feat(deploy): support the index-only option when syncing from/to local copies 2024-02-28 08:02:35 +00:00
proletarius101
86524ea56d feat(deploy): support index-only mode in libcloud 2024-02-28 08:02:35 +00:00
proletarius101
69a508e762 feat(deploy): support --index-only option for serverwebroot deployments 2024-02-28 08:02:35 +00:00
proletarius101
3eb51a1711 refactor: revert changes to the full sync mode for s3cmd 2024-02-28 08:02:35 +00:00
proletarius101
0021c76a44 feat(update_awsbucket_s3cmd): support the index-only flag 2024-02-28 08:02:35 +00:00
proletarius101
fb0041bdc1 feat: add --index-only flag 2024-02-28 08:02:35 +00:00
5 changed files with 971 additions and 247 deletions

View File

@ -183,7 +183,7 @@
#
# serverwebroot:
# - url: 'me@b.az:/srv/fdroid'
# indexOnly: true
# index_only: true
# When running fdroid processes on a remote server, it is possible to
@ -209,7 +209,7 @@
# servergitmirrors:
# - url: https://github.com/user/repo
# - url: https://gitlab.com/user/repo
# indexOnly: true
# index_only: true
# Most git hosting services have hard size limits for each git repo.
# `fdroid deploy` will delete the git history when the git mirror repo

View File

@ -29,6 +29,7 @@
# libraries here as they will become a requirement for all commands.
import difflib
from typing import List
import git
import glob
import io
@ -3996,7 +3997,7 @@ def get_app_display_name(app):
return app.get('AutoName') or app['id']
def local_rsync(options, fromdir, todir):
def local_rsync(options, from_paths: List[str], todir: str):
"""Rsync method for local to local copying of things.
This is an rsync wrapper with all the settings for safe use within
@ -4013,8 +4014,8 @@ def local_rsync(options, fromdir, todir):
rsyncargs += ['--verbose']
if options.quiet:
rsyncargs += ['--quiet']
logging.debug(' '.join(rsyncargs + [fromdir, todir]))
if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
raise FDroidException()

View File

@ -25,10 +25,14 @@ import re
import subprocess
import time
import urllib
from typing import Dict, Optional, List
from git import Repo
import yaml
from argparse import ArgumentParser
from argparse import ArgumentParser, Namespace
import logging
import shutil
import git
from pathlib import Path
from . import _
from . import common
@ -36,7 +40,7 @@ from . import index
from .exception import FDroidException
config = None
options = None
options: Optional[Namespace] = None
start_timestamp = time.gmtime()
GIT_BRANCH = 'master'
@ -61,27 +65,19 @@ INDEX_FILES = [
]
def _get_index_excludes(repo_section):
def _get_index_file_paths(base_dir):
"""Return the list of files to be synced last, since they finalize the deploy.
The process of pushing all the new packages to the various
services can take a while. So the index files should be updated
last. That ensures that the package files are available when the
client learns about them from the new index files.
"""
indexes = [
os.path.join(repo_section, 'entry.jar'),
os.path.join(repo_section, 'entry.json'),
os.path.join(repo_section, 'entry.json.asc'),
os.path.join(repo_section, 'index-v1.jar'),
os.path.join(repo_section, 'index-v1.json'),
os.path.join(repo_section, 'index-v1.json.asc'),
os.path.join(repo_section, 'index-v2.json'),
os.path.join(repo_section, 'index-v2.json.asc'),
os.path.join(repo_section, 'index.jar'),
os.path.join(repo_section, 'index.xml'),
]
return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
def _get_index_excludes(base_dir):
indexes = _get_index_file_paths(base_dir)
index_excludes = []
for f in indexes:
index_excludes.append('--exclude')
@ -89,7 +85,16 @@ def _get_index_excludes(repo_section):
return index_excludes
def update_awsbucket(repo_section):
def _get_index_includes(base_dir):
indexes = _get_index_file_paths(base_dir)
index_includes = []
for f in indexes:
index_includes.append('--include')
index_includes.append(f)
return index_includes
def update_awsbucket(repo_section, is_index_only=False):
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
The contents of that subdir of the
@ -101,12 +106,12 @@ def update_awsbucket(repo_section):
+ config['awsbucket'] + '"')
if common.set_command_in_config('s3cmd'):
update_awsbucket_s3cmd(repo_section)
update_awsbucket_s3cmd(repo_section, is_index_only)
else:
update_awsbucket_libcloud(repo_section)
update_awsbucket_libcloud(repo_section, is_index_only)
def update_awsbucket_s3cmd(repo_section):
def update_awsbucket_s3cmd(repo_section, is_index_only=False):
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
The upload is done in multiple passes to reduce the chance of
@ -149,6 +154,22 @@ def update_awsbucket_s3cmd(repo_section):
s3cmd_sync += ['--quiet']
s3url = s3bucketurl + '/fdroid/'
if is_index_only:
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
.format(path=repo_section, url=s3url))
sync_indexes_flags = []
sync_indexes_flags.extend(_get_index_includes(repo_section))
sync_indexes_flags.append('--delete-removed')
sync_indexes_flags.append('--delete-after')
if options.no_checksum:
sync_indexes_flags.append('--no-check-md5')
else:
sync_indexes_flags.append('--check-md5')
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
if returncode != 0:
raise FDroidException()
else:
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
logging.debug(_('Running first pass with MD5 checking disabled'))
excludes = _get_index_excludes(repo_section)
@ -178,7 +199,7 @@ def update_awsbucket_s3cmd(repo_section):
raise FDroidException()
def update_awsbucket_libcloud(repo_section):
def update_awsbucket_libcloud(repo_section, is_index_only=False):
"""No summary.
Upload the contents of the directory `repo_section` (including
@ -221,10 +242,16 @@ def update_awsbucket_libcloud(repo_section):
if obj.name.startswith(upload_dir + '/'):
objs[obj.name] = obj
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
for name in files:
if is_index_only:
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
files_to_upload = list(set(files_to_upload) & set(index_files))
else:
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
for file_to_upload in files_to_upload:
upload = False
file_to_upload = os.path.join(root, name)
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
if object_name not in objs:
upload = True
@ -308,14 +335,22 @@ def update_serverwebroot(serverwebroot, repo_section):
elif config and config.get('identity_file'):
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
url = serverwebroot['url']
is_index_only = serverwebroot.get('index_only', False)
logging.info('rsyncing ' + repo_section + ' to ' + url)
if is_index_only:
rsyncargs += _get_index_file_paths(repo_section)
rsyncargs += [f'{url}/{repo_section}/']
logging.info(rsyncargs)
if subprocess.call(rsyncargs) != 0:
raise FDroidException()
else:
excludes = _get_index_excludes(repo_section)
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
raise FDroidException()
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
raise FDroidException()
# upload "current version" symlinks if requested
if config and config.get('make_current_version_link') and repo_section == 'repo':
if config['make_current_version_link'] and repo_section == 'repo':
links_to_upload = []
for f in glob.glob('*.apk') \
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
@ -364,10 +399,11 @@ def sync_from_localcopy(repo_section, local_copy_dir):
"""
logging.info('Syncing from local_copy_dir to this repo.')
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
common.local_rsync(options,
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
[os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
repo_section.rstrip('/') + '/')
offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
@ -385,7 +421,7 @@ def update_localcopy(repo_section, local_copy_dir):
"""
# local_copy_dir is guaranteed to have a trailing slash in main() below
common.local_rsync(options, repo_section, local_copy_dir)
common.local_rsync(options, [repo_section], local_copy_dir)
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
if os.path.isdir(os.path.join(offline_copy, '.git')):
@ -415,16 +451,17 @@ def update_servergitmirrors(servergitmirrors, repo_section):
transparency log.
"""
import git
from clint.textui import progress
if config.get('local_copy_dir') \
and not config.get('sync_from_local_copy_dir'):
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
return
workspace_dir = Path(os.getcwd())
# right now we support only 'repo' git-mirroring
if repo_section == 'repo':
git_mirror_path = 'git-mirror'
git_mirror_path = workspace_dir / 'git-mirror'
dotgit = os.path.join(git_mirror_path, '.git')
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
git_repodir = os.path.join(git_fdroiddir, repo_section)
@ -445,11 +482,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
shutil.rmtree(archive_path, ignore_errors=True)
# rsync is very particular about trailing slashes
common.local_rsync(options,
repo_section.rstrip('/') + '/',
git_repodir.rstrip('/') + '/')
# use custom SSH command if identity_file specified
ssh_cmd = 'ssh -oBatchMode=yes'
if options.identity_file is not None:
@ -457,28 +489,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
elif 'identity_file' in config:
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
enabled_remotes = []
for d in servergitmirrors:
remote_url = d['url']
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
enabled_remotes.append(name)
r = git.remote.Remote(repo, name)
if r in repo.remotes:
r = repo.remote(name)
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
r.set_url(remote_url)
else:
repo.create_remote(name, remote_url)
logging.info('Mirroring to: ' + remote_url)
# sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror')
repo.git.add(all=True)
logging.debug('Committing all files into git mirror')
repo.index.commit("fdroidserver git-mirror")
if options.verbose:
progressbar = progress.Bar()
@ -490,8 +500,81 @@ def update_servergitmirrors(servergitmirrors, repo_section):
else:
progress = None
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
enabled_remotes = []
for d in servergitmirrors:
is_index_only = d.get('index_only', False)
# Use a separate branch for the index only mode as it needs a different set of files to commit
if is_index_only:
local_branch_name = 'index_only'
else:
local_branch_name = 'full'
if local_branch_name in repo.heads:
repo.git.switch(local_branch_name)
else:
repo.git.switch('--orphan', local_branch_name)
# trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash
if is_index_only:
files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
else:
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
upload_to_servergitmirror(mirror_config=d,
local_repo=repo,
enabled_remotes=enabled_remotes,
repo_section=repo_section,
is_index_only=is_index_only,
fdroid_dir=git_fdroiddir,
git_mirror_path=str(git_mirror_path),
ssh_cmd=ssh_cmd,
progress=progress)
if progress:
progressbar.done()
def upload_to_servergitmirror(mirror_config: Dict[str, str],
local_repo: Repo,
enabled_remotes: List[str],
repo_section: str,
is_index_only: bool,
fdroid_dir: str,
git_mirror_path: str,
ssh_cmd: str,
progress: git.RemoteProgress) -> None:
remote_branch_name = GIT_BRANCH
local_branch_name = local_repo.active_branch.name
remote_url = mirror_config['url']
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
enabled_remotes.append(name)
r = git.remote.Remote(local_repo, name)
if r in local_repo.remotes:
r = local_repo.remote(name)
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
r.set_url(remote_url)
else:
local_repo.create_remote(name, remote_url)
logging.info('Mirroring to: ' + remote_url)
logging.info("git status:", local_repo.git.status())
if is_index_only:
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
else:
# sadly index.add don't allow the --all parameter
logging.debug('Adding all files to git mirror')
local_repo.git.add(all=True)
logging.debug('Committing files into git mirror')
local_repo.index.commit("fdroidserver git-mirror")
# only deploy to GitLab Artifacts if too big for GitLab Pages
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
gitlab_ci_job_name = 'pages'
else:
gitlab_ci_job_name = 'GitLab Artifacts'
@ -502,11 +585,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
)
# push for every remote. This will overwrite the git history
for remote in repo.remotes:
if remote.name not in enabled_remotes:
repo.delete_remote(remote)
continue
# push. This will overwrite the git history
remote = local_repo.remote(name)
if remote.name == 'gitlab':
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
@ -526,13 +606,13 @@ def update_servergitmirrors(servergitmirrors, repo_section):
default_flow_style=False,
)
repo.git.add(all=True)
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
local_repo.index.add(['.gitlab-ci.yml'])
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
logging.debug(_('Pushing to {url}').format(url=remote.url))
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
logging.info(_('Pushing to {url}').format(url=remote.url))
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
pushinfos = remote.push(
GIT_BRANCH, force=True, set_upstream=True, progress=progress
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
)
for pushinfo in pushinfos:
if pushinfo.flags & (git.remote.PushInfo.ERROR
@ -548,9 +628,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
else:
logging.debug(remote.url + ': ' + pushinfo.summary)
if progress:
progressbar.done()
def upload_to_android_observatory(repo_section):
import requests

View File

@ -321,11 +321,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
os.chdir(repo_basedir)
if os.path.isdir(git_mirror_repodir):
common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
if os.path.isdir(git_mirror_metadatadir):
common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
if os.path.isdir(git_mirror_statsdir):
common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
ssh_private_key_file = _ssh_key_from_debug_keystore()
# this is needed for GitPython to find the SSH key
@ -427,9 +427,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
cwd=repo_basedir,
)
common.local_rsync(
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
)
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata")

View File

@ -10,6 +10,8 @@ import unittest
from pathlib import Path
from unittest import mock
import git
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
@ -60,6 +62,10 @@ class DeployTest(unittest.TestCase):
url1 = Path('url1/fdroid')
url1.mkdir(parents=True)
# setup parameters for this test run
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk0 = url0 / fake_apk
dest_apk1 = url1 / fake_apk
self.assertFalse(dest_apk0.is_file())
@ -96,13 +102,54 @@ class DeployTest(unittest.TestCase):
fake_apk = repo / 'fake.apk'
with fake_apk.open('w') as fp:
fp.write('not an APK, but has the right filename')
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename')
url = Path('url')
url.mkdir()
dest_apk = url / fake_apk
# setup parameters for this test run
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk
dest_index = Path(url) / fake_index
self.assertFalse(dest_apk.is_file())
self.assertFalse(dest_index.is_file())
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
self.assertTrue(dest_apk.is_file())
self.assertTrue(dest_index.is_file())
def test_update_serverwebroot_in_index_only_mode(self):
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir()
fake_apk = repo / 'fake.apk'
with fake_apk.open('w') as fp:
fp.write('not an APK, but has the right filename')
for i in fdroidserver.deploy.INDEX_FILES:
fake_index = repo / i
with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename')
url = Path('url')
url.mkdir()
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = False
dest_apk = Path(url) / fake_apk
dest_index = Path(url) / fake_index
self.assertFalse(dest_apk.is_file())
self.assertFalse(dest_index.is_file())
fdroidserver.deploy.update_serverwebroot(
{'url': str(url), 'index_only': True}, 'repo'
)
self.assertFalse(dest_apk.is_file())
self.assertTrue(dest_index.is_file())
@mock.patch.dict(os.environ, clear=True)
def test_update_serverwebroot_no_rsync_error(self):
@ -118,7 +165,8 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.config = {'make_current_version_link': True}
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid"
repo_section = 'repo'
@ -202,6 +250,89 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['make_current_version_link'] = True
url = "example.com:/var/www/fdroid"
repo_section = 'repo'
# setup function for asserting subprocess.call invocations
call_iteration = 0
def update_server_webroot_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(
cmd,
[
'rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'repo/entry.jar',
'repo/entry.json',
'repo/entry.json.asc',
'repo/index-v1.jar',
'repo/index-v1.json',
'repo/index-v1.json.asc',
'repo/index-v2.json',
'repo/index-v2.json.asc',
'repo/index.jar',
'repo/index.xml',
'example.com:/var/www/fdroid/repo/',
],
)
elif call_iteration == 1:
self.assertListEqual(
cmd,
[
'rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'repo',
url,
],
)
elif call_iteration == 2:
self.assertListEqual(
cmd,
[
'rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'Sym.apk',
'Sym.apk.asc',
'Sym.apk.sig',
'example.com:/var/www/fdroid',
],
)
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('repo')
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.deploy.update_serverwebroot(
{'url': url, 'index_only': True}, repo_section
)
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
def test_update_serverwebroot_with_id_file(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
@ -210,7 +341,8 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.options.verbose = True
fdroidserver.deploy.options.quiet = False
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config = {'identity_file': './id_rsa'}
fdroidserver.deploy.config['identity_file'] = './id_rsa'
fdroidserver.deploy.config['make_current_version_link'] = False
url = "example.com:/var/www/fdroid"
repo_section = 'archive'
@ -280,10 +412,79 @@ class DeployTest(unittest.TestCase):
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_chcksum = False
fdroidserver.deploy.options.verbose = True
fdroidserver.deploy.options.quiet = False
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.config['identity_file'] = './id_rsa'
fdroidserver.deploy.config['make_current_version_link'] = False
url = "example.com:/var/www/fdroid"
repo_section = 'archive'
# setup function for asserting subprocess.call invocations
call_iteration = 0
def update_server_webroot_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(
cmd,
[
'rsync',
'--archive',
'--delete-after',
'--safe-links',
'--verbose',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'],
'archive/entry.jar',
'archive/entry.json',
'archive/entry.json.asc',
'archive/index-v1.jar',
'archive/index-v1.json',
'archive/index-v1.json.asc',
'archive/index-v2.json',
'archive/index-v2.json.asc',
'archive/index.jar',
'archive/index.xml',
"example.com:/var/www/fdroid/archive/",
],
)
elif call_iteration == 1:
self.assertListEqual(
cmd,
[
'rsync',
'--archive',
'--delete-after',
'--safe-links',
'--verbose',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.deploy.config['identity_file'],
"example.com:/var/www/fdroid/archive/",
],
)
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.deploy.update_serverwebroot(
{'url': url, 'index_only': True}, repo_section
)
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
@unittest.skipIf(
not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
)
def test_upload_to_virustotal(self):
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.verbose = True
virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
@ -300,25 +501,239 @@ class DeployTest(unittest.TestCase):
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
)
def test_update_servergitmirrors(self):
def test_update_awsbucket_s3cmd(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["servergitmirrors"] = []
fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
repo_section = 'repo'
# setup function for asserting subprocess.call invocations
update_servergitmirrors_call_iteration = 0
remote_push_call_iteration = 0
call_iteration = 0
def update_awsbucket_s3cmd_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'info',
f"s3://{fdroidserver.deploy.config['awsbucket']}",
],
)
elif call_iteration == 1:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'sync',
'--acl-public',
'--quiet',
'--exclude',
'repo/entry.jar',
'--exclude',
'repo/entry.json',
'--exclude',
'repo/entry.json.asc',
'--exclude',
'repo/index-v1.jar',
'--exclude',
'repo/index-v1.json',
'--exclude',
'repo/index-v1.json.asc',
'--exclude',
'repo/index-v2.json',
'--exclude',
'repo/index-v2.json.asc',
'--exclude',
'repo/index.jar',
'--exclude',
'repo/index.xml',
'--no-check-md5',
'--skip-existing',
repo_section,
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
],
)
elif call_iteration == 2:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'sync',
'--acl-public',
'--quiet',
'--exclude',
'repo/entry.jar',
'--exclude',
'repo/entry.json',
'--exclude',
'repo/entry.json.asc',
'--exclude',
'repo/index-v1.jar',
'--exclude',
'repo/index-v1.json',
'--exclude',
'repo/index-v1.json.asc',
'--exclude',
'repo/index-v2.json',
'--exclude',
'repo/index-v2.json.asc',
'--exclude',
'repo/index.jar',
'--exclude',
'repo/index.xml',
'--no-check-md5',
repo_section,
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
],
)
elif call_iteration == 3:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'sync',
'--acl-public',
'--quiet',
'--delete-removed',
'--delete-after',
'--no-check-md5',
repo_section,
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
],
)
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('repo')
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
def test_update_awsbucket_s3cmd_in_index_only_mode(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
repo_section = 'repo'
# setup function for asserting subprocess.call invocations
call_iteration = 0
def update_awsbucket_s3cmd_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'info',
f"s3://{fdroidserver.deploy.config['awsbucket']}",
],
)
elif call_iteration == 1:
self.assertListEqual(
cmd,
[
's3cmd',
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
'sync',
'--acl-public',
'--quiet',
'--include',
'repo/entry.jar',
'--include',
'repo/entry.json',
'--include',
'repo/entry.json.asc',
'--include',
'repo/index-v1.jar',
'--include',
'repo/index-v1.json',
'--include',
'repo/index-v1.json.asc',
'--include',
'repo/index-v2.json',
'--include',
'repo/index-v2.json.asc',
'--include',
'repo/index.jar',
'--include',
'repo/index.xml',
'--delete-removed',
'--delete-after',
'--no-check-md5',
repo_section,
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
],
)
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('repo')
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
fdroidserver.deploy.update_awsbucket_s3cmd(
repo_section, index_only=True
)
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
def test_update_awsbucket_libcloud(self):
from libcloud.storage.base import Container
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
repo_section = 'repo'
os.chdir(self.testdir)
repo = Path('repo')
@ -330,54 +745,285 @@ class DeployTest(unittest.TestCase):
with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename')
def update_servergitmirrors_call(cmd):
nonlocal update_servergitmirrors_call_iteration
if update_servergitmirrors_call_iteration == 0:
self.assertListEqual(
cmd,
[
'rsync',
'--recursive',
'--safe-links',
'--times',
'--perms',
'--one-file-system',
'--delete',
'--chmod=Da+rx,Fa-x,a+r,u+w',
'--quiet',
'repo/',
"git-mirror/fdroid/repo/",
],
)
else:
self.fail('unexpected subprocess.call invocation')
update_servergitmirrors_call_iteration += 1
return 0
def remote_push_call(ref, force=False, set_upstream=False, **_args):
nonlocal remote_push_call_iteration
if remote_push_call_iteration == 0:
self.assertEqual([ref, force, set_upstream], ['master', True, True])
else:
self.fail('unexpected git.Remote.push invocation')
remote_push_call_iteration += 1
return []
with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
with mock.patch(
'git.Remote.push', side_effect=remote_push_call
) as mock_remote_push:
mock_remote_push.return_value = []
'libcloud.storage.drivers.s3.S3StorageDriver'
) as mock_driver_class:
mock_driver = mock_driver_class.return_value
mock_container = mock.MagicMock(spec=Container)
mock_container.list_objects.return_value = [
mock.MagicMock(name='Sym.apk'),
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
]
mock_driver.get_container.return_value = mock_container
mock_driver.upload_object_via_stream.return_value = None
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
mock_driver.get_container.assert_called_once_with(
container_name=fdroidserver.deploy.config["awsbucket"]
)
mock_container.list_objects.assert_called_once_with()
files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
calls = [
mock.call(
iterator=mock.ANY,
container=mock_container,
object_name=file,
extra={'acl': 'public-read'},
)
for file in files_to_upload
]
mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
assert mock_driver.upload_object_via_stream.call_count == 2
def test_update_awsbucket_libcloud_in_index_only_mode(self):
from libcloud.storage.base import Container
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.no_checksum = True
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["awsbucket"] = "bucket"
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
repo_section = 'repo'
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir(parents=True)
fake_apk = repo / 'Sym.apk'
with fake_apk.open('w') as fp:
fp.write('not an APK, but has the right filename')
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename')
with mock.patch(
'libcloud.storage.drivers.s3.S3StorageDriver'
) as mock_driver_class:
mock_driver = mock_driver_class.return_value
mock_container = mock.MagicMock(spec=Container)
mock_container.list_objects.return_value = [
mock.MagicMock(name='Sym.apk'),
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
]
mock_driver.get_container.return_value = mock_container
mock_driver.upload_object_via_stream.return_value = None
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
mock_driver.get_container.assert_called_once_with(
container_name=fdroidserver.deploy.config["awsbucket"]
)
mock_container.list_objects.assert_called_once_with()
files_to_upload = ['fdroid/repo/entry.jar']
calls = [
mock.call(
iterator=mock.ANY,
container=mock_container,
object_name=file,
extra={'acl': 'public-read'},
)
for file in files_to_upload
]
mock_driver.upload_object_via_stream.assert_has_calls(
calls,
any_order=False,
)
assert mock_driver.upload_object_via_stream.call_count == 1
def test_update_servergitmirrors(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
os.chdir(self.testdir)
repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
remote_repo = Path(self.testdir) / 'remote'
remote_repo.mkdir(parents=True)
remote_git_repo = git.Repo.init(
remote_repo, initial_branch=initial_branch, bare=True
)
fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir(parents=True)
fake_apk = 'Sym.apk'
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
for filename in fake_files:
fake_file = repo / filename
with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename')
fdroidserver.deploy.update_servergitmirrors(
[{'url': 'https://github.com/user/repo'}], repo_section
fdroidserver.deploy.config["servergitmirrors"], repo_section
)
self.assertEqual(
update_servergitmirrors_call_iteration,
1,
'expected 1 invocations of subprocess.call',
verify_repo = remote_git_repo.clone(
Path(self.testdir) / 'verify',
)
self.assertEqual(
remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
for filename in fake_files:
remote_file = f"fdroid/{repo_section}/{filename}"
self.assertIsNotNone(verify_repo.working_tree_dir)
if verify_repo.working_tree_dir is not None:
self.assertTrue(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
def test_update_servergitmirrors_in_index_only_mode(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
os.chdir(self.testdir)
repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
remote_repo = Path(self.testdir) / 'remote'
remote_repo.mkdir(parents=True)
remote_git_repo = git.Repo.init(
remote_repo, initial_branch=initial_branch, bare=True
)
fdroidserver.deploy.config["servergitmirrors"] = [
{"url": str(remote_repo), "index_only": True}
]
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir(parents=True)
fake_apk = 'Sym.apk'
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
for filename in fake_files:
fake_file = repo / filename
with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename')
fdroidserver.deploy.update_servergitmirrors(
fdroidserver.deploy.config["servergitmirrors"], repo_section
)
verify_repo = remote_git_repo.clone(
Path(self.testdir) / 'verify',
)
for filename in fdroidserver.deploy.INDEX_FILES:
remote_file = f"fdroid/{repo_section}/{filename}"
self.assertIsNotNone(verify_repo.working_tree_dir)
if verify_repo.working_tree_dir is not None:
self.assertTrue(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
# Should not have the APK file
remote_file = f"fdroid/{repo_section}/{fake_apk}"
if verify_repo.working_tree_dir is not None:
self.assertFalse(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
def test_upload_to_servergitmirror_in_index_only_mode(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.identity_file = None
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
repo_section = 'repo'
initial_branch = fdroidserver.deploy.GIT_BRANCH
os.chdir(self.testdir)
local_git_repo_path = Path(self.testdir) / 'local'
local_git_repo = git.Repo.init(
local_git_repo_path, initial_branch=initial_branch
)
fdroid_dir = local_git_repo_path / 'fdroid'
repo_dir = fdroid_dir / repo_section
repo_dir.mkdir(parents=True)
fake_apk = 'Sym.apk'
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
for filename in fake_files:
fake_file = repo_dir / filename
with fake_file.open('w') as fp:
fp.write('not a real one, but has the right filename')
# The remote repo must be a bare repo to allow being pushed to
remote_git_repo_dir = Path(self.testdir) / 'remote'
remote_git_repo = git.Repo.init(
remote_git_repo_dir, initial_branch=initial_branch, bare=True
)
mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
enabled_remotes = []
ssh_cmd = 'ssh -oBatchMode=yes'
fdroidserver.deploy.upload_to_servergitmirror(
mirror_config=mirror_config,
local_repo=local_git_repo,
enabled_remotes=enabled_remotes,
repo_section=repo_section,
is_index_only=mirror_config['index_only'],
fdroid_dir=str(fdroid_dir),
git_mirror_path=str(local_git_repo_path),
ssh_cmd=ssh_cmd,
progress=git.RemoteProgress(),
)
verify_repo = remote_git_repo.clone(
Path(self.testdir) / 'verify',
)
for filename in fdroidserver.deploy.INDEX_FILES:
remote_file = f"fdroid/{repo_section}/{filename}"
self.assertIsNotNone(verify_repo.working_tree_dir)
if verify_repo.working_tree_dir is not None:
self.assertTrue(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)
# Should not have the APK file
remote_file = f"fdroid/{repo_section}/{fake_apk}"
if verify_repo.working_tree_dir is not None:
self.assertFalse(
(Path(verify_repo.working_tree_dir) / remote_file).exists()
)