mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
Compare commits
67 Commits
8ec3a2a3db
...
da5ff1c928
Author | SHA1 | Date | |
---|---|---|---|
|
da5ff1c928 | ||
|
dc3fd0bdd1 | ||
|
c26267a7f9 | ||
|
48c48596a8 | ||
|
5ff9a19072 | ||
|
78b6e53e42 | ||
|
40ecaff6fe | ||
|
5aeece612f | ||
|
abdaced2a5 | ||
|
3833bc7d6e | ||
|
7d123b5578 | ||
|
59fdaa321c | ||
|
9441d0e849 | ||
|
a09552207d | ||
|
10c06b8b44 | ||
|
d2a867a018 | ||
|
a19fc1d607 | ||
|
cd2a0fbf29 | ||
|
04c191e4d8 | ||
|
5fe1fa9dd3 | ||
|
092a8dda2e | ||
|
4f09ba2bff | ||
|
c148ea3d8e | ||
|
d79ac8e2dc | ||
|
5aef1da1fc | ||
|
e966ac2efe | ||
|
1cc23b20c8 | ||
|
a5740f364c | ||
|
4f2ec8f2e3 | ||
|
5a46b062f4 | ||
|
649bb5b1bc | ||
|
462786bb37 | ||
|
31f9919073 | ||
|
cd23cf659b | ||
|
d9984dbeea | ||
|
6ddf241713 | ||
|
5e0980b98a | ||
|
d2ada6e418 | ||
|
96d1f4190f | ||
|
2d267e6cbd | ||
|
513b223caa | ||
|
6271de094e | ||
|
51b88b784b | ||
|
7022eabe84 | ||
|
1e259cb043 | ||
|
b3c15038da | ||
|
775316747c | ||
|
4a793cd67a | ||
|
e05bc9c1db | ||
|
6b1c5bae0f | ||
|
b4b4b20b4c | ||
|
d814291577 | ||
|
707b91e1eb | ||
|
34395b0932 | ||
|
aa99f27afb | ||
|
a853d6027e | ||
|
738d7dbf4e | ||
|
450b0d9a84 | ||
|
7775aff1f2 | ||
|
aa54495388 | ||
|
35952418a2 | ||
|
8e6996b25d | ||
|
86524ea56d | ||
|
69a508e762 | ||
|
3eb51a1711 | ||
|
0021c76a44 | ||
|
fb0041bdc1 |
@ -183,7 +183,7 @@
|
|||||||
#
|
#
|
||||||
# serverwebroot:
|
# serverwebroot:
|
||||||
# - url: 'me@b.az:/srv/fdroid'
|
# - url: 'me@b.az:/srv/fdroid'
|
||||||
# indexOnly: true
|
# index_only: true
|
||||||
|
|
||||||
|
|
||||||
# When running fdroid processes on a remote server, it is possible to
|
# When running fdroid processes on a remote server, it is possible to
|
||||||
@ -209,7 +209,7 @@
|
|||||||
# servergitmirrors:
|
# servergitmirrors:
|
||||||
# - url: https://github.com/user/repo
|
# - url: https://github.com/user/repo
|
||||||
# - url: https://gitlab.com/user/repo
|
# - url: https://gitlab.com/user/repo
|
||||||
# indexOnly: true
|
# index_only: true
|
||||||
|
|
||||||
# Most git hosting services have hard size limits for each git repo.
|
# Most git hosting services have hard size limits for each git repo.
|
||||||
# `fdroid deploy` will delete the git history when the git mirror repo
|
# `fdroid deploy` will delete the git history when the git mirror repo
|
||||||
|
@ -29,6 +29,7 @@
|
|||||||
# libraries here as they will become a requirement for all commands.
|
# libraries here as they will become a requirement for all commands.
|
||||||
|
|
||||||
import difflib
|
import difflib
|
||||||
|
from typing import List
|
||||||
import git
|
import git
|
||||||
import glob
|
import glob
|
||||||
import io
|
import io
|
||||||
@ -3996,7 +3997,7 @@ def get_app_display_name(app):
|
|||||||
return app.get('AutoName') or app['id']
|
return app.get('AutoName') or app['id']
|
||||||
|
|
||||||
|
|
||||||
def local_rsync(options, fromdir, todir):
|
def local_rsync(options, from_paths: List[str], todir: str):
|
||||||
"""Rsync method for local to local copying of things.
|
"""Rsync method for local to local copying of things.
|
||||||
|
|
||||||
This is an rsync wrapper with all the settings for safe use within
|
This is an rsync wrapper with all the settings for safe use within
|
||||||
@ -4013,8 +4014,8 @@ def local_rsync(options, fromdir, todir):
|
|||||||
rsyncargs += ['--verbose']
|
rsyncargs += ['--verbose']
|
||||||
if options.quiet:
|
if options.quiet:
|
||||||
rsyncargs += ['--quiet']
|
rsyncargs += ['--quiet']
|
||||||
logging.debug(' '.join(rsyncargs + [fromdir, todir]))
|
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
|
||||||
if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
|
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
|
||||||
raise FDroidException()
|
raise FDroidException()
|
||||||
|
|
||||||
|
|
||||||
|
@ -25,10 +25,14 @@ import re
|
|||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
import urllib
|
import urllib
|
||||||
|
from typing import Dict, Optional, List
|
||||||
|
from git import Repo
|
||||||
import yaml
|
import yaml
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser, Namespace
|
||||||
import logging
|
import logging
|
||||||
import shutil
|
import shutil
|
||||||
|
import git
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from . import _
|
from . import _
|
||||||
from . import common
|
from . import common
|
||||||
@ -36,7 +40,7 @@ from . import index
|
|||||||
from .exception import FDroidException
|
from .exception import FDroidException
|
||||||
|
|
||||||
config = None
|
config = None
|
||||||
options = None
|
options: Optional[Namespace] = None
|
||||||
start_timestamp = time.gmtime()
|
start_timestamp = time.gmtime()
|
||||||
|
|
||||||
GIT_BRANCH = 'master'
|
GIT_BRANCH = 'master'
|
||||||
@ -61,27 +65,19 @@ INDEX_FILES = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _get_index_excludes(repo_section):
|
def _get_index_file_paths(base_dir):
|
||||||
"""Return the list of files to be synced last, since they finalize the deploy.
|
"""Return the list of files to be synced last, since they finalize the deploy.
|
||||||
|
|
||||||
The process of pushing all the new packages to the various
|
The process of pushing all the new packages to the various
|
||||||
services can take a while. So the index files should be updated
|
services can take a while. So the index files should be updated
|
||||||
last. That ensures that the package files are available when the
|
last. That ensures that the package files are available when the
|
||||||
client learns about them from the new index files.
|
client learns about them from the new index files.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
indexes = [
|
return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
|
||||||
os.path.join(repo_section, 'entry.jar'),
|
|
||||||
os.path.join(repo_section, 'entry.json'),
|
|
||||||
os.path.join(repo_section, 'entry.json.asc'),
|
def _get_index_excludes(base_dir):
|
||||||
os.path.join(repo_section, 'index-v1.jar'),
|
indexes = _get_index_file_paths(base_dir)
|
||||||
os.path.join(repo_section, 'index-v1.json'),
|
|
||||||
os.path.join(repo_section, 'index-v1.json.asc'),
|
|
||||||
os.path.join(repo_section, 'index-v2.json'),
|
|
||||||
os.path.join(repo_section, 'index-v2.json.asc'),
|
|
||||||
os.path.join(repo_section, 'index.jar'),
|
|
||||||
os.path.join(repo_section, 'index.xml'),
|
|
||||||
]
|
|
||||||
index_excludes = []
|
index_excludes = []
|
||||||
for f in indexes:
|
for f in indexes:
|
||||||
index_excludes.append('--exclude')
|
index_excludes.append('--exclude')
|
||||||
@ -89,7 +85,16 @@ def _get_index_excludes(repo_section):
|
|||||||
return index_excludes
|
return index_excludes
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket(repo_section):
|
def _get_index_includes(base_dir):
|
||||||
|
indexes = _get_index_file_paths(base_dir)
|
||||||
|
index_includes = []
|
||||||
|
for f in indexes:
|
||||||
|
index_includes.append('--include')
|
||||||
|
index_includes.append(f)
|
||||||
|
return index_includes
|
||||||
|
|
||||||
|
|
||||||
|
def update_awsbucket(repo_section, is_index_only=False):
|
||||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||||
|
|
||||||
The contents of that subdir of the
|
The contents of that subdir of the
|
||||||
@ -101,12 +106,12 @@ def update_awsbucket(repo_section):
|
|||||||
+ config['awsbucket'] + '"')
|
+ config['awsbucket'] + '"')
|
||||||
|
|
||||||
if common.set_command_in_config('s3cmd'):
|
if common.set_command_in_config('s3cmd'):
|
||||||
update_awsbucket_s3cmd(repo_section)
|
update_awsbucket_s3cmd(repo_section, is_index_only)
|
||||||
else:
|
else:
|
||||||
update_awsbucket_libcloud(repo_section)
|
update_awsbucket_libcloud(repo_section, is_index_only)
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket_s3cmd(repo_section):
|
def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||||
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
||||||
|
|
||||||
The upload is done in multiple passes to reduce the chance of
|
The upload is done in multiple passes to reduce the chance of
|
||||||
@ -149,36 +154,52 @@ def update_awsbucket_s3cmd(repo_section):
|
|||||||
s3cmd_sync += ['--quiet']
|
s3cmd_sync += ['--quiet']
|
||||||
|
|
||||||
s3url = s3bucketurl + '/fdroid/'
|
s3url = s3bucketurl + '/fdroid/'
|
||||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
|
||||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
|
||||||
excludes = _get_index_excludes(repo_section)
|
|
||||||
returncode = subprocess.call(
|
|
||||||
s3cmd_sync
|
|
||||||
+ excludes
|
|
||||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
|
||||||
)
|
|
||||||
if returncode != 0:
|
|
||||||
raise FDroidException()
|
|
||||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
|
||||||
returncode = subprocess.call(
|
|
||||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
|
||||||
)
|
|
||||||
if returncode != 0:
|
|
||||||
raise FDroidException()
|
|
||||||
|
|
||||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
if is_index_only:
|
||||||
.format(path=repo_section, url=s3url))
|
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||||
s3cmd_sync.append('--delete-removed')
|
.format(path=repo_section, url=s3url))
|
||||||
s3cmd_sync.append('--delete-after')
|
sync_indexes_flags = []
|
||||||
if options.no_checksum:
|
sync_indexes_flags.extend(_get_index_includes(repo_section))
|
||||||
s3cmd_sync.append('--no-check-md5')
|
sync_indexes_flags.append('--delete-removed')
|
||||||
|
sync_indexes_flags.append('--delete-after')
|
||||||
|
if options.no_checksum:
|
||||||
|
sync_indexes_flags.append('--no-check-md5')
|
||||||
|
else:
|
||||||
|
sync_indexes_flags.append('--check-md5')
|
||||||
|
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
|
||||||
|
if returncode != 0:
|
||||||
|
raise FDroidException()
|
||||||
else:
|
else:
|
||||||
s3cmd_sync.append('--check-md5')
|
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||||
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||||
raise FDroidException()
|
excludes = _get_index_excludes(repo_section)
|
||||||
|
returncode = subprocess.call(
|
||||||
|
s3cmd_sync
|
||||||
|
+ excludes
|
||||||
|
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||||
|
)
|
||||||
|
if returncode != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||||
|
returncode = subprocess.call(
|
||||||
|
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
||||||
|
)
|
||||||
|
if returncode != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
|
||||||
|
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||||
|
.format(path=repo_section, url=s3url))
|
||||||
|
s3cmd_sync.append('--delete-removed')
|
||||||
|
s3cmd_sync.append('--delete-after')
|
||||||
|
if options.no_checksum:
|
||||||
|
s3cmd_sync.append('--no-check-md5')
|
||||||
|
else:
|
||||||
|
s3cmd_sync.append('--check-md5')
|
||||||
|
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket_libcloud(repo_section):
|
def update_awsbucket_libcloud(repo_section, is_index_only=False):
|
||||||
"""No summary.
|
"""No summary.
|
||||||
|
|
||||||
Upload the contents of the directory `repo_section` (including
|
Upload the contents of the directory `repo_section` (including
|
||||||
@ -221,47 +242,53 @@ def update_awsbucket_libcloud(repo_section):
|
|||||||
if obj.name.startswith(upload_dir + '/'):
|
if obj.name.startswith(upload_dir + '/'):
|
||||||
objs[obj.name] = obj
|
objs[obj.name] = obj
|
||||||
|
|
||||||
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
|
if is_index_only:
|
||||||
for name in files:
|
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
||||||
upload = False
|
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||||
file_to_upload = os.path.join(root, name)
|
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||||
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
|
||||||
if object_name not in objs:
|
else:
|
||||||
|
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||||
|
|
||||||
|
for file_to_upload in files_to_upload:
|
||||||
|
upload = False
|
||||||
|
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
||||||
|
if object_name not in objs:
|
||||||
|
upload = True
|
||||||
|
else:
|
||||||
|
obj = objs.pop(object_name)
|
||||||
|
if obj.size != os.path.getsize(file_to_upload):
|
||||||
upload = True
|
upload = True
|
||||||
else:
|
else:
|
||||||
obj = objs.pop(object_name)
|
# if the sizes match, then compare by MD5
|
||||||
if obj.size != os.path.getsize(file_to_upload):
|
md5 = hashlib.md5() # nosec AWS uses MD5
|
||||||
|
with open(file_to_upload, 'rb') as f:
|
||||||
|
while True:
|
||||||
|
data = f.read(8192)
|
||||||
|
if not data:
|
||||||
|
break
|
||||||
|
md5.update(data)
|
||||||
|
if obj.hash != md5.hexdigest():
|
||||||
|
s3url = 's3://' + awsbucket + '/' + obj.name
|
||||||
|
logging.info(' deleting ' + s3url)
|
||||||
|
if not driver.delete_object(obj):
|
||||||
|
logging.warning('Could not delete ' + s3url)
|
||||||
upload = True
|
upload = True
|
||||||
else:
|
|
||||||
# if the sizes match, then compare by MD5
|
|
||||||
md5 = hashlib.md5() # nosec AWS uses MD5
|
|
||||||
with open(file_to_upload, 'rb') as f:
|
|
||||||
while True:
|
|
||||||
data = f.read(8192)
|
|
||||||
if not data:
|
|
||||||
break
|
|
||||||
md5.update(data)
|
|
||||||
if obj.hash != md5.hexdigest():
|
|
||||||
s3url = 's3://' + awsbucket + '/' + obj.name
|
|
||||||
logging.info(' deleting ' + s3url)
|
|
||||||
if not driver.delete_object(obj):
|
|
||||||
logging.warning('Could not delete ' + s3url)
|
|
||||||
upload = True
|
|
||||||
|
|
||||||
if upload:
|
if upload:
|
||||||
logging.debug(' uploading "' + file_to_upload + '"...')
|
logging.debug(' uploading "' + file_to_upload + '"...')
|
||||||
extra = {'acl': 'public-read'}
|
extra = {'acl': 'public-read'}
|
||||||
if file_to_upload.endswith('.sig'):
|
if file_to_upload.endswith('.sig'):
|
||||||
extra['content_type'] = 'application/pgp-signature'
|
extra['content_type'] = 'application/pgp-signature'
|
||||||
elif file_to_upload.endswith('.asc'):
|
elif file_to_upload.endswith('.asc'):
|
||||||
extra['content_type'] = 'application/pgp-signature'
|
extra['content_type'] = 'application/pgp-signature'
|
||||||
logging.info(' uploading ' + os.path.relpath(file_to_upload)
|
logging.info(' uploading ' + os.path.relpath(file_to_upload)
|
||||||
+ ' to s3://' + awsbucket + '/' + object_name)
|
+ ' to s3://' + awsbucket + '/' + object_name)
|
||||||
with open(file_to_upload, 'rb') as iterator:
|
with open(file_to_upload, 'rb') as iterator:
|
||||||
obj = driver.upload_object_via_stream(iterator=iterator,
|
obj = driver.upload_object_via_stream(iterator=iterator,
|
||||||
container=container,
|
container=container,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
extra=extra)
|
extra=extra)
|
||||||
# delete the remnants in the bucket, they do not exist locally
|
# delete the remnants in the bucket, they do not exist locally
|
||||||
while objs:
|
while objs:
|
||||||
object_name, obj = objs.popitem()
|
object_name, obj = objs.popitem()
|
||||||
@ -308,22 +335,30 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||||||
elif config and config.get('identity_file'):
|
elif config and config.get('identity_file'):
|
||||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||||
url = serverwebroot['url']
|
url = serverwebroot['url']
|
||||||
|
is_index_only = serverwebroot.get('index_only', False)
|
||||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||||
excludes = _get_index_excludes(repo_section)
|
if is_index_only:
|
||||||
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
rsyncargs += _get_index_file_paths(repo_section)
|
||||||
raise FDroidException()
|
rsyncargs += [f'{url}/{repo_section}/']
|
||||||
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
logging.info(rsyncargs)
|
||||||
raise FDroidException()
|
if subprocess.call(rsyncargs) != 0:
|
||||||
# upload "current version" symlinks if requested
|
raise FDroidException()
|
||||||
if config and config.get('make_current_version_link') and repo_section == 'repo':
|
else:
|
||||||
links_to_upload = []
|
excludes = _get_index_excludes(repo_section)
|
||||||
for f in glob.glob('*.apk') \
|
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
||||||
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
raise FDroidException()
|
||||||
if os.path.islink(f):
|
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
||||||
links_to_upload.append(f)
|
raise FDroidException()
|
||||||
if len(links_to_upload) > 0:
|
# upload "current version" symlinks if requested
|
||||||
if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
|
if config['make_current_version_link'] and repo_section == 'repo':
|
||||||
raise FDroidException()
|
links_to_upload = []
|
||||||
|
for f in glob.glob('*.apk') \
|
||||||
|
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
||||||
|
if os.path.islink(f):
|
||||||
|
links_to_upload.append(f)
|
||||||
|
if len(links_to_upload) > 0:
|
||||||
|
if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
|
||||||
|
|
||||||
def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
|
def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
|
||||||
@ -364,10 +399,11 @@ def sync_from_localcopy(repo_section, local_copy_dir):
|
|||||||
|
|
||||||
"""
|
"""
|
||||||
logging.info('Syncing from local_copy_dir to this repo.')
|
logging.info('Syncing from local_copy_dir to this repo.')
|
||||||
|
|
||||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||||
# make sure both paths have exactly one trailing slash
|
# make sure both paths have exactly one trailing slash
|
||||||
common.local_rsync(options,
|
common.local_rsync(options,
|
||||||
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
|
[os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
|
||||||
repo_section.rstrip('/') + '/')
|
repo_section.rstrip('/') + '/')
|
||||||
|
|
||||||
offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
|
offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
|
||||||
@ -385,7 +421,7 @@ def update_localcopy(repo_section, local_copy_dir):
|
|||||||
|
|
||||||
"""
|
"""
|
||||||
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
||||||
common.local_rsync(options, repo_section, local_copy_dir)
|
common.local_rsync(options, [repo_section], local_copy_dir)
|
||||||
|
|
||||||
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
|
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
|
||||||
if os.path.isdir(os.path.join(offline_copy, '.git')):
|
if os.path.isdir(os.path.join(offline_copy, '.git')):
|
||||||
@ -415,16 +451,17 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
transparency log.
|
transparency log.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import git
|
|
||||||
from clint.textui import progress
|
from clint.textui import progress
|
||||||
if config.get('local_copy_dir') \
|
if config.get('local_copy_dir') \
|
||||||
and not config.get('sync_from_local_copy_dir'):
|
and not config.get('sync_from_local_copy_dir'):
|
||||||
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
|
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
|
||||||
return
|
return
|
||||||
|
|
||||||
|
workspace_dir = Path(os.getcwd())
|
||||||
|
|
||||||
# right now we support only 'repo' git-mirroring
|
# right now we support only 'repo' git-mirroring
|
||||||
if repo_section == 'repo':
|
if repo_section == 'repo':
|
||||||
git_mirror_path = 'git-mirror'
|
git_mirror_path = workspace_dir / 'git-mirror'
|
||||||
dotgit = os.path.join(git_mirror_path, '.git')
|
dotgit = os.path.join(git_mirror_path, '.git')
|
||||||
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
|
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
|
||||||
git_repodir = os.path.join(git_fdroiddir, repo_section)
|
git_repodir = os.path.join(git_fdroiddir, repo_section)
|
||||||
@ -445,11 +482,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
|
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
|
||||||
shutil.rmtree(archive_path, ignore_errors=True)
|
shutil.rmtree(archive_path, ignore_errors=True)
|
||||||
|
|
||||||
# rsync is very particular about trailing slashes
|
|
||||||
common.local_rsync(options,
|
|
||||||
repo_section.rstrip('/') + '/',
|
|
||||||
git_repodir.rstrip('/') + '/')
|
|
||||||
|
|
||||||
# use custom SSH command if identity_file specified
|
# use custom SSH command if identity_file specified
|
||||||
ssh_cmd = 'ssh -oBatchMode=yes'
|
ssh_cmd = 'ssh -oBatchMode=yes'
|
||||||
if options.identity_file is not None:
|
if options.identity_file is not None:
|
||||||
@ -457,28 +489,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
elif 'identity_file' in config:
|
elif 'identity_file' in config:
|
||||||
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
|
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
|
||||||
|
|
||||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
|
||||||
|
|
||||||
enabled_remotes = []
|
|
||||||
for d in servergitmirrors:
|
|
||||||
remote_url = d['url']
|
|
||||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
|
||||||
enabled_remotes.append(name)
|
|
||||||
r = git.remote.Remote(repo, name)
|
|
||||||
if r in repo.remotes:
|
|
||||||
r = repo.remote(name)
|
|
||||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
|
||||||
r.set_url(remote_url)
|
|
||||||
else:
|
|
||||||
repo.create_remote(name, remote_url)
|
|
||||||
logging.info('Mirroring to: ' + remote_url)
|
|
||||||
|
|
||||||
# sadly index.add don't allow the --all parameter
|
|
||||||
logging.debug('Adding all files to git mirror')
|
|
||||||
repo.git.add(all=True)
|
|
||||||
logging.debug('Committing all files into git mirror')
|
|
||||||
repo.index.commit("fdroidserver git-mirror")
|
|
||||||
|
|
||||||
if options.verbose:
|
if options.verbose:
|
||||||
progressbar = progress.Bar()
|
progressbar = progress.Bar()
|
||||||
|
|
||||||
@ -490,68 +500,135 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
else:
|
else:
|
||||||
progress = None
|
progress = None
|
||||||
|
|
||||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
|
||||||
gitlab_ci_job_name = 'pages'
|
|
||||||
else:
|
|
||||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
|
||||||
logging.warning(
|
|
||||||
_(
|
|
||||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
|
||||||
)
|
|
||||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
|
||||||
)
|
|
||||||
|
|
||||||
# push for every remote. This will overwrite the git history
|
enabled_remotes = []
|
||||||
for remote in repo.remotes:
|
for d in servergitmirrors:
|
||||||
if remote.name not in enabled_remotes:
|
is_index_only = d.get('index_only', False)
|
||||||
repo.delete_remote(remote)
|
|
||||||
continue
|
|
||||||
if remote.name == 'gitlab':
|
|
||||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
|
||||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
|
||||||
yaml.dump(
|
|
||||||
{
|
|
||||||
gitlab_ci_job_name: {
|
|
||||||
'script': [
|
|
||||||
'mkdir .public',
|
|
||||||
'cp -r * .public/',
|
|
||||||
'mv .public public',
|
|
||||||
],
|
|
||||||
'artifacts': {'paths': ['public']},
|
|
||||||
'variables': {'GIT_DEPTH': 1},
|
|
||||||
}
|
|
||||||
},
|
|
||||||
fp,
|
|
||||||
default_flow_style=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
repo.git.add(all=True)
|
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
||||||
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
if is_index_only:
|
||||||
|
local_branch_name = 'index_only'
|
||||||
|
else:
|
||||||
|
local_branch_name = 'full'
|
||||||
|
if local_branch_name in repo.heads:
|
||||||
|
repo.git.switch(local_branch_name)
|
||||||
|
else:
|
||||||
|
repo.git.switch('--orphan', local_branch_name)
|
||||||
|
|
||||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||||
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
# make sure both paths have exactly one trailing slash
|
||||||
pushinfos = remote.push(
|
if is_index_only:
|
||||||
GIT_BRANCH, force=True, set_upstream=True, progress=progress
|
files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
|
||||||
)
|
else:
|
||||||
for pushinfo in pushinfos:
|
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
|
||||||
if pushinfo.flags & (git.remote.PushInfo.ERROR
|
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
|
||||||
| git.remote.PushInfo.REJECTED
|
|
||||||
| git.remote.PushInfo.REMOTE_FAILURE
|
|
||||||
| git.remote.PushInfo.REMOTE_REJECTED):
|
|
||||||
# Show potentially useful messages from git remote
|
|
||||||
for line in progress.other_lines:
|
|
||||||
if line.startswith('remote:'):
|
|
||||||
logging.debug(line)
|
|
||||||
raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
|
|
||||||
+ ' ' + pushinfo.summary)
|
|
||||||
else:
|
|
||||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
|
||||||
|
|
||||||
|
upload_to_servergitmirror(mirror_config=d,
|
||||||
|
local_repo=repo,
|
||||||
|
enabled_remotes=enabled_remotes,
|
||||||
|
repo_section=repo_section,
|
||||||
|
is_index_only=is_index_only,
|
||||||
|
fdroid_dir=git_fdroiddir,
|
||||||
|
git_mirror_path=str(git_mirror_path),
|
||||||
|
ssh_cmd=ssh_cmd,
|
||||||
|
progress=progress)
|
||||||
if progress:
|
if progress:
|
||||||
progressbar.done()
|
progressbar.done()
|
||||||
|
|
||||||
|
|
||||||
|
def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||||
|
local_repo: Repo,
|
||||||
|
enabled_remotes: List[str],
|
||||||
|
repo_section: str,
|
||||||
|
is_index_only: bool,
|
||||||
|
fdroid_dir: str,
|
||||||
|
git_mirror_path: str,
|
||||||
|
ssh_cmd: str,
|
||||||
|
progress: git.RemoteProgress) -> None:
|
||||||
|
remote_branch_name = GIT_BRANCH
|
||||||
|
local_branch_name = local_repo.active_branch.name
|
||||||
|
|
||||||
|
remote_url = mirror_config['url']
|
||||||
|
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||||
|
enabled_remotes.append(name)
|
||||||
|
r = git.remote.Remote(local_repo, name)
|
||||||
|
if r in local_repo.remotes:
|
||||||
|
r = local_repo.remote(name)
|
||||||
|
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||||
|
r.set_url(remote_url)
|
||||||
|
else:
|
||||||
|
local_repo.create_remote(name, remote_url)
|
||||||
|
logging.info('Mirroring to: ' + remote_url)
|
||||||
|
|
||||||
|
logging.info("git status:", local_repo.git.status())
|
||||||
|
if is_index_only:
|
||||||
|
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||||
|
else:
|
||||||
|
# sadly index.add don't allow the --all parameter
|
||||||
|
logging.debug('Adding all files to git mirror')
|
||||||
|
local_repo.git.add(all=True)
|
||||||
|
|
||||||
|
logging.debug('Committing files into git mirror')
|
||||||
|
local_repo.index.commit("fdroidserver git-mirror")
|
||||||
|
|
||||||
|
|
||||||
|
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||||
|
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||||
|
gitlab_ci_job_name = 'pages'
|
||||||
|
else:
|
||||||
|
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||||
|
logging.warning(
|
||||||
|
_(
|
||||||
|
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||||
|
)
|
||||||
|
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||||
|
)
|
||||||
|
|
||||||
|
# push. This will overwrite the git history
|
||||||
|
remote = local_repo.remote(name)
|
||||||
|
if remote.name == 'gitlab':
|
||||||
|
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||||
|
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||||
|
yaml.dump(
|
||||||
|
{
|
||||||
|
gitlab_ci_job_name: {
|
||||||
|
'script': [
|
||||||
|
'mkdir .public',
|
||||||
|
'cp -r * .public/',
|
||||||
|
'mv .public public',
|
||||||
|
],
|
||||||
|
'artifacts': {'paths': ['public']},
|
||||||
|
'variables': {'GIT_DEPTH': 1},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
fp,
|
||||||
|
default_flow_style=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
local_repo.index.add(['.gitlab-ci.yml'])
|
||||||
|
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||||
|
|
||||||
|
logging.info(_('Pushing to {url}').format(url=remote.url))
|
||||||
|
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||||
|
pushinfos = remote.push(
|
||||||
|
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
|
||||||
|
)
|
||||||
|
for pushinfo in pushinfos:
|
||||||
|
if pushinfo.flags & (git.remote.PushInfo.ERROR
|
||||||
|
| git.remote.PushInfo.REJECTED
|
||||||
|
| git.remote.PushInfo.REMOTE_FAILURE
|
||||||
|
| git.remote.PushInfo.REMOTE_REJECTED):
|
||||||
|
# Show potentially useful messages from git remote
|
||||||
|
for line in progress.other_lines:
|
||||||
|
if line.startswith('remote:'):
|
||||||
|
logging.debug(line)
|
||||||
|
raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
|
||||||
|
+ ' ' + pushinfo.summary)
|
||||||
|
else:
|
||||||
|
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||||
|
|
||||||
|
|
||||||
def upload_to_android_observatory(repo_section):
|
def upload_to_android_observatory(repo_section):
|
||||||
import requests
|
import requests
|
||||||
requests # stop unused import warning
|
requests # stop unused import warning
|
||||||
|
@ -321,11 +321,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||||||
|
|
||||||
os.chdir(repo_basedir)
|
os.chdir(repo_basedir)
|
||||||
if os.path.isdir(git_mirror_repodir):
|
if os.path.isdir(git_mirror_repodir):
|
||||||
common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
|
common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
|
||||||
if os.path.isdir(git_mirror_metadatadir):
|
if os.path.isdir(git_mirror_metadatadir):
|
||||||
common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
|
common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
|
||||||
if os.path.isdir(git_mirror_statsdir):
|
if os.path.isdir(git_mirror_statsdir):
|
||||||
common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
|
common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
|
||||||
|
|
||||||
ssh_private_key_file = _ssh_key_from_debug_keystore()
|
ssh_private_key_file = _ssh_key_from_debug_keystore()
|
||||||
# this is needed for GitPython to find the SSH key
|
# this is needed for GitPython to find the SSH key
|
||||||
@ -427,9 +427,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||||||
cwd=repo_basedir,
|
cwd=repo_basedir,
|
||||||
)
|
)
|
||||||
common.local_rsync(
|
common.local_rsync(
|
||||||
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
|
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
|
||||||
)
|
)
|
||||||
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
|
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
|
||||||
mirror_git_repo.git.add(all=True)
|
mirror_git_repo.git.add(all=True)
|
||||||
mirror_git_repo.index.commit("update app metadata")
|
mirror_git_repo.index.commit("update app metadata")
|
||||||
|
|
||||||
|
@ -10,6 +10,8 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
import git
|
||||||
|
|
||||||
localmodule = os.path.realpath(
|
localmodule = os.path.realpath(
|
||||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||||
)
|
)
|
||||||
@ -60,6 +62,10 @@ class DeployTest(unittest.TestCase):
|
|||||||
url1 = Path('url1/fdroid')
|
url1 = Path('url1/fdroid')
|
||||||
url1.mkdir(parents=True)
|
url1.mkdir(parents=True)
|
||||||
|
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||||
|
|
||||||
dest_apk0 = url0 / fake_apk
|
dest_apk0 = url0 / fake_apk
|
||||||
dest_apk1 = url1 / fake_apk
|
dest_apk1 = url1 / fake_apk
|
||||||
self.assertFalse(dest_apk0.is_file())
|
self.assertFalse(dest_apk0.is_file())
|
||||||
@ -96,13 +102,54 @@ class DeployTest(unittest.TestCase):
|
|||||||
fake_apk = repo / 'fake.apk'
|
fake_apk = repo / 'fake.apk'
|
||||||
with fake_apk.open('w') as fp:
|
with fake_apk.open('w') as fp:
|
||||||
fp.write('not an APK, but has the right filename')
|
fp.write('not an APK, but has the right filename')
|
||||||
|
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||||
|
with fake_index.open('w') as fp:
|
||||||
|
fp.write('not an index, but has the right filename')
|
||||||
url = Path('url')
|
url = Path('url')
|
||||||
url.mkdir()
|
url.mkdir()
|
||||||
|
|
||||||
dest_apk = url / fake_apk
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||||
|
|
||||||
|
dest_apk = Path(url) / fake_apk
|
||||||
|
dest_index = Path(url) / fake_index
|
||||||
self.assertFalse(dest_apk.is_file())
|
self.assertFalse(dest_apk.is_file())
|
||||||
|
self.assertFalse(dest_index.is_file())
|
||||||
|
|
||||||
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
||||||
self.assertTrue(dest_apk.is_file())
|
self.assertTrue(dest_apk.is_file())
|
||||||
|
self.assertTrue(dest_index.is_file())
|
||||||
|
|
||||||
|
def test_update_serverwebroot_in_index_only_mode(self):
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
repo = Path('repo')
|
||||||
|
repo.mkdir()
|
||||||
|
fake_apk = repo / 'fake.apk'
|
||||||
|
with fake_apk.open('w') as fp:
|
||||||
|
fp.write('not an APK, but has the right filename')
|
||||||
|
for i in fdroidserver.deploy.INDEX_FILES:
|
||||||
|
fake_index = repo / i
|
||||||
|
with fake_index.open('w') as fp:
|
||||||
|
fp.write('not an index, but has the right filename')
|
||||||
|
url = Path('url')
|
||||||
|
url.mkdir()
|
||||||
|
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||||
|
|
||||||
|
dest_apk = Path(url) / fake_apk
|
||||||
|
dest_index = Path(url) / fake_index
|
||||||
|
self.assertFalse(dest_apk.is_file())
|
||||||
|
self.assertFalse(dest_index.is_file())
|
||||||
|
|
||||||
|
fdroidserver.deploy.update_serverwebroot(
|
||||||
|
{'url': str(url), 'index_only': True}, 'repo'
|
||||||
|
)
|
||||||
|
self.assertFalse(dest_apk.is_file())
|
||||||
|
self.assertTrue(dest_index.is_file())
|
||||||
|
|
||||||
@mock.patch.dict(os.environ, clear=True)
|
@mock.patch.dict(os.environ, clear=True)
|
||||||
def test_update_serverwebroot_no_rsync_error(self):
|
def test_update_serverwebroot_no_rsync_error(self):
|
||||||
@ -118,7 +165,8 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.deploy.options.identity_file = None
|
fdroidserver.deploy.options.identity_file = None
|
||||||
fdroidserver.deploy.options.verbose = False
|
fdroidserver.deploy.options.verbose = False
|
||||||
fdroidserver.deploy.options.quiet = True
|
fdroidserver.deploy.options.quiet = True
|
||||||
fdroidserver.deploy.config = {'make_current_version_link': True}
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||||
url = "example.com:/var/www/fdroid"
|
url = "example.com:/var/www/fdroid"
|
||||||
repo_section = 'repo'
|
repo_section = 'repo'
|
||||||
|
|
||||||
@ -202,6 +250,89 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||||
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
|
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
|
||||||
|
|
||||||
|
def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.no_checksum = True
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||||
|
url = "example.com:/var/www/fdroid"
|
||||||
|
repo_section = 'repo'
|
||||||
|
|
||||||
|
# setup function for asserting subprocess.call invocations
|
||||||
|
call_iteration = 0
|
||||||
|
|
||||||
|
def update_server_webroot_call(cmd):
|
||||||
|
nonlocal call_iteration
|
||||||
|
if call_iteration == 0:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
'rsync',
|
||||||
|
'--archive',
|
||||||
|
'--delete-after',
|
||||||
|
'--safe-links',
|
||||||
|
'--quiet',
|
||||||
|
'repo/entry.jar',
|
||||||
|
'repo/entry.json',
|
||||||
|
'repo/entry.json.asc',
|
||||||
|
'repo/index-v1.jar',
|
||||||
|
'repo/index-v1.json',
|
||||||
|
'repo/index-v1.json.asc',
|
||||||
|
'repo/index-v2.json',
|
||||||
|
'repo/index-v2.json.asc',
|
||||||
|
'repo/index.jar',
|
||||||
|
'repo/index.xml',
|
||||||
|
'example.com:/var/www/fdroid/repo/',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 1:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
'rsync',
|
||||||
|
'--archive',
|
||||||
|
'--delete-after',
|
||||||
|
'--safe-links',
|
||||||
|
'--quiet',
|
||||||
|
'repo',
|
||||||
|
url,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 2:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
'rsync',
|
||||||
|
'--archive',
|
||||||
|
'--delete-after',
|
||||||
|
'--safe-links',
|
||||||
|
'--quiet',
|
||||||
|
'Sym.apk',
|
||||||
|
'Sym.apk.asc',
|
||||||
|
'Sym.apk.sig',
|
||||||
|
'example.com:/var/www/fdroid',
|
||||||
|
],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.fail('unexpected subprocess.call invocation')
|
||||||
|
call_iteration += 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||||
|
os.mkdir('repo')
|
||||||
|
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||||
|
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||||
|
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||||
|
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||||
|
fdroidserver.deploy.update_serverwebroot(
|
||||||
|
{'url': url, 'index_only': True}, repo_section
|
||||||
|
)
|
||||||
|
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||||
|
|
||||||
def test_update_serverwebroot_with_id_file(self):
|
def test_update_serverwebroot_with_id_file(self):
|
||||||
# setup parameters for this test run
|
# setup parameters for this test run
|
||||||
fdroidserver.deploy.options = mock.Mock()
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
@ -210,7 +341,8 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.deploy.options.verbose = True
|
fdroidserver.deploy.options.verbose = True
|
||||||
fdroidserver.deploy.options.quiet = False
|
fdroidserver.deploy.options.quiet = False
|
||||||
fdroidserver.deploy.options.identity_file = None
|
fdroidserver.deploy.options.identity_file = None
|
||||||
fdroidserver.deploy.config = {'identity_file': './id_rsa'}
|
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||||
url = "example.com:/var/www/fdroid"
|
url = "example.com:/var/www/fdroid"
|
||||||
repo_section = 'archive'
|
repo_section = 'archive'
|
||||||
|
|
||||||
@ -280,10 +412,79 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||||
|
|
||||||
|
def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.no_chcksum = False
|
||||||
|
fdroidserver.deploy.options.verbose = True
|
||||||
|
fdroidserver.deploy.options.quiet = False
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||||
|
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||||
|
url = "example.com:/var/www/fdroid"
|
||||||
|
repo_section = 'archive'
|
||||||
|
|
||||||
|
# setup function for asserting subprocess.call invocations
|
||||||
|
call_iteration = 0
|
||||||
|
|
||||||
|
def update_server_webroot_call(cmd):
|
||||||
|
nonlocal call_iteration
|
||||||
|
if call_iteration == 0:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
'rsync',
|
||||||
|
'--archive',
|
||||||
|
'--delete-after',
|
||||||
|
'--safe-links',
|
||||||
|
'--verbose',
|
||||||
|
'-e',
|
||||||
|
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||||
|
+ fdroidserver.deploy.config['identity_file'],
|
||||||
|
'archive/entry.jar',
|
||||||
|
'archive/entry.json',
|
||||||
|
'archive/entry.json.asc',
|
||||||
|
'archive/index-v1.jar',
|
||||||
|
'archive/index-v1.json',
|
||||||
|
'archive/index-v1.json.asc',
|
||||||
|
'archive/index-v2.json',
|
||||||
|
'archive/index-v2.json.asc',
|
||||||
|
'archive/index.jar',
|
||||||
|
'archive/index.xml',
|
||||||
|
"example.com:/var/www/fdroid/archive/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 1:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
'rsync',
|
||||||
|
'--archive',
|
||||||
|
'--delete-after',
|
||||||
|
'--safe-links',
|
||||||
|
'--verbose',
|
||||||
|
'-e',
|
||||||
|
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||||
|
+ fdroidserver.deploy.config['identity_file'],
|
||||||
|
"example.com:/var/www/fdroid/archive/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.fail('unexpected subprocess.call invocation')
|
||||||
|
call_iteration += 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||||
|
fdroidserver.deploy.update_serverwebroot(
|
||||||
|
{'url': url, 'index_only': True}, repo_section
|
||||||
|
)
|
||||||
|
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||||
|
|
||||||
@unittest.skipIf(
|
@unittest.skipIf(
|
||||||
not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
|
not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
|
||||||
)
|
)
|
||||||
def test_upload_to_virustotal(self):
|
def test_upload_to_virustotal(self):
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
fdroidserver.deploy.options.verbose = True
|
fdroidserver.deploy.options.verbose = True
|
||||||
virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
|
virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
|
||||||
fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
|
fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
|
||||||
@ -300,25 +501,239 @@ class DeployTest(unittest.TestCase):
|
|||||||
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_update_servergitmirrors(self):
|
def test_update_awsbucket_s3cmd(self):
|
||||||
# setup parameters for this test run
|
# setup parameters for this test run
|
||||||
fdroidserver.deploy.options = mock.Mock()
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
fdroidserver.deploy.options.identity_file = None
|
fdroidserver.deploy.options.no_checksum = True
|
||||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
|
||||||
fdroidserver.deploy.options.verbose = False
|
fdroidserver.deploy.options.verbose = False
|
||||||
fdroidserver.deploy.options.quiet = True
|
fdroidserver.deploy.options.quiet = True
|
||||||
fdroidserver.deploy.options.index_only = False
|
|
||||||
|
|
||||||
config = {}
|
config = {}
|
||||||
fdroidserver.common.fill_config_defaults(config)
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
fdroidserver.deploy.config = config
|
fdroidserver.deploy.config = config
|
||||||
fdroidserver.deploy.config["servergitmirrors"] = []
|
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||||
|
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||||
|
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||||
|
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||||
|
|
||||||
repo_section = 'repo'
|
repo_section = 'repo'
|
||||||
|
|
||||||
# setup function for asserting subprocess.call invocations
|
# setup function for asserting subprocess.call invocations
|
||||||
update_servergitmirrors_call_iteration = 0
|
call_iteration = 0
|
||||||
remote_push_call_iteration = 0
|
|
||||||
|
def update_awsbucket_s3cmd_call(cmd):
|
||||||
|
nonlocal call_iteration
|
||||||
|
if call_iteration == 0:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'info',
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 1:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'sync',
|
||||||
|
'--acl-public',
|
||||||
|
'--quiet',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v2.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v2.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index.xml',
|
||||||
|
'--no-check-md5',
|
||||||
|
'--skip-existing',
|
||||||
|
repo_section,
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 2:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'sync',
|
||||||
|
'--acl-public',
|
||||||
|
'--quiet',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/entry.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v1.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v2.json',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index-v2.json.asc',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index.jar',
|
||||||
|
'--exclude',
|
||||||
|
'repo/index.xml',
|
||||||
|
'--no-check-md5',
|
||||||
|
repo_section,
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 3:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'sync',
|
||||||
|
'--acl-public',
|
||||||
|
'--quiet',
|
||||||
|
'--delete-removed',
|
||||||
|
'--delete-after',
|
||||||
|
'--no-check-md5',
|
||||||
|
repo_section,
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.fail('unexpected subprocess.call invocation')
|
||||||
|
call_iteration += 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||||
|
os.mkdir('repo')
|
||||||
|
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||||
|
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||||
|
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||||
|
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||||
|
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
|
||||||
|
self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
|
||||||
|
|
||||||
|
def test_update_awsbucket_s3cmd_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.no_checksum = True
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||||
|
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||||
|
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||||
|
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
|
||||||
|
# setup function for asserting subprocess.call invocations
|
||||||
|
call_iteration = 0
|
||||||
|
|
||||||
|
def update_awsbucket_s3cmd_call(cmd):
|
||||||
|
nonlocal call_iteration
|
||||||
|
if call_iteration == 0:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'info',
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
elif call_iteration == 1:
|
||||||
|
self.assertListEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
's3cmd',
|
||||||
|
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||||
|
'sync',
|
||||||
|
'--acl-public',
|
||||||
|
'--quiet',
|
||||||
|
'--include',
|
||||||
|
'repo/entry.jar',
|
||||||
|
'--include',
|
||||||
|
'repo/entry.json',
|
||||||
|
'--include',
|
||||||
|
'repo/entry.json.asc',
|
||||||
|
'--include',
|
||||||
|
'repo/index-v1.jar',
|
||||||
|
'--include',
|
||||||
|
'repo/index-v1.json',
|
||||||
|
'--include',
|
||||||
|
'repo/index-v1.json.asc',
|
||||||
|
'--include',
|
||||||
|
'repo/index-v2.json',
|
||||||
|
'--include',
|
||||||
|
'repo/index-v2.json.asc',
|
||||||
|
'--include',
|
||||||
|
'repo/index.jar',
|
||||||
|
'--include',
|
||||||
|
'repo/index.xml',
|
||||||
|
'--delete-removed',
|
||||||
|
'--delete-after',
|
||||||
|
'--no-check-md5',
|
||||||
|
repo_section,
|
||||||
|
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.fail('unexpected subprocess.call invocation')
|
||||||
|
call_iteration += 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||||
|
os.mkdir('repo')
|
||||||
|
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||||
|
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||||
|
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||||
|
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||||
|
fdroidserver.deploy.update_awsbucket_s3cmd(
|
||||||
|
repo_section, index_only=True
|
||||||
|
)
|
||||||
|
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||||
|
|
||||||
|
def test_update_awsbucket_libcloud(self):
|
||||||
|
from libcloud.storage.base import Container
|
||||||
|
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.no_checksum = True
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||||
|
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||||
|
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||||
|
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
|
||||||
os.chdir(self.testdir)
|
os.chdir(self.testdir)
|
||||||
repo = Path('repo')
|
repo = Path('repo')
|
||||||
@ -330,56 +745,287 @@ class DeployTest(unittest.TestCase):
|
|||||||
with fake_index.open('w') as fp:
|
with fake_index.open('w') as fp:
|
||||||
fp.write('not an index, but has the right filename')
|
fp.write('not an index, but has the right filename')
|
||||||
|
|
||||||
def update_servergitmirrors_call(cmd):
|
with mock.patch(
|
||||||
nonlocal update_servergitmirrors_call_iteration
|
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||||
if update_servergitmirrors_call_iteration == 0:
|
) as mock_driver_class:
|
||||||
self.assertListEqual(
|
mock_driver = mock_driver_class.return_value
|
||||||
cmd,
|
mock_container = mock.MagicMock(spec=Container)
|
||||||
[
|
mock_container.list_objects.return_value = [
|
||||||
'rsync',
|
mock.MagicMock(name='Sym.apk'),
|
||||||
'--recursive',
|
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
|
||||||
'--safe-links',
|
]
|
||||||
'--times',
|
|
||||||
'--perms',
|
|
||||||
'--one-file-system',
|
|
||||||
'--delete',
|
|
||||||
'--chmod=Da+rx,Fa-x,a+r,u+w',
|
|
||||||
'--quiet',
|
|
||||||
'repo/',
|
|
||||||
"git-mirror/fdroid/repo/",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.fail('unexpected subprocess.call invocation')
|
|
||||||
update_servergitmirrors_call_iteration += 1
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def remote_push_call(ref, force=False, set_upstream=False, **_args):
|
mock_driver.get_container.return_value = mock_container
|
||||||
nonlocal remote_push_call_iteration
|
mock_driver.upload_object_via_stream.return_value = None
|
||||||
if remote_push_call_iteration == 0:
|
|
||||||
self.assertEqual([ref, force, set_upstream], ['master', True, True])
|
|
||||||
else:
|
|
||||||
self.fail('unexpected git.Remote.push invocation')
|
|
||||||
remote_push_call_iteration += 1
|
|
||||||
return []
|
|
||||||
|
|
||||||
with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
|
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
|
||||||
with mock.patch(
|
|
||||||
'git.Remote.push', side_effect=remote_push_call
|
mock_driver.get_container.assert_called_once_with(
|
||||||
) as mock_remote_push:
|
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||||
mock_remote_push.return_value = []
|
)
|
||||||
fdroidserver.deploy.update_servergitmirrors(
|
mock_container.list_objects.assert_called_once_with()
|
||||||
[{'url': 'https://github.com/user/repo'}], repo_section
|
files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
|
||||||
|
calls = [
|
||||||
|
mock.call(
|
||||||
|
iterator=mock.ANY,
|
||||||
|
container=mock_container,
|
||||||
|
object_name=file,
|
||||||
|
extra={'acl': 'public-read'},
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
for file in files_to_upload
|
||||||
update_servergitmirrors_call_iteration,
|
]
|
||||||
1,
|
mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
|
||||||
'expected 1 invocations of subprocess.call',
|
assert mock_driver.upload_object_via_stream.call_count == 2
|
||||||
|
|
||||||
|
def test_update_awsbucket_libcloud_in_index_only_mode(self):
|
||||||
|
from libcloud.storage.base import Container
|
||||||
|
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.no_checksum = True
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||||
|
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||||
|
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||||
|
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
repo = Path('repo')
|
||||||
|
repo.mkdir(parents=True)
|
||||||
|
fake_apk = repo / 'Sym.apk'
|
||||||
|
with fake_apk.open('w') as fp:
|
||||||
|
fp.write('not an APK, but has the right filename')
|
||||||
|
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||||
|
with fake_index.open('w') as fp:
|
||||||
|
fp.write('not an index, but has the right filename')
|
||||||
|
|
||||||
|
with mock.patch(
|
||||||
|
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||||
|
) as mock_driver_class:
|
||||||
|
mock_driver = mock_driver_class.return_value
|
||||||
|
mock_container = mock.MagicMock(spec=Container)
|
||||||
|
mock_container.list_objects.return_value = [
|
||||||
|
mock.MagicMock(name='Sym.apk'),
|
||||||
|
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
|
||||||
|
]
|
||||||
|
|
||||||
|
mock_driver.get_container.return_value = mock_container
|
||||||
|
mock_driver.upload_object_via_stream.return_value = None
|
||||||
|
|
||||||
|
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
|
||||||
|
|
||||||
|
mock_driver.get_container.assert_called_once_with(
|
||||||
|
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||||
|
)
|
||||||
|
mock_container.list_objects.assert_called_once_with()
|
||||||
|
files_to_upload = ['fdroid/repo/entry.jar']
|
||||||
|
calls = [
|
||||||
|
mock.call(
|
||||||
|
iterator=mock.ANY,
|
||||||
|
container=mock_container,
|
||||||
|
object_name=file,
|
||||||
|
extra={'acl': 'public-read'},
|
||||||
|
)
|
||||||
|
for file in files_to_upload
|
||||||
|
]
|
||||||
|
mock_driver.upload_object_via_stream.assert_has_calls(
|
||||||
|
calls,
|
||||||
|
any_order=False,
|
||||||
|
)
|
||||||
|
assert mock_driver.upload_object_via_stream.call_count == 1
|
||||||
|
|
||||||
|
def test_update_servergitmirrors(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||||
|
|
||||||
|
remote_repo = Path(self.testdir) / 'remote'
|
||||||
|
remote_repo.mkdir(parents=True)
|
||||||
|
remote_git_repo = git.Repo.init(
|
||||||
|
remote_repo, initial_branch=initial_branch, bare=True
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
|
||||||
remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
repo = Path('repo')
|
||||||
|
repo.mkdir(parents=True)
|
||||||
|
fake_apk = 'Sym.apk'
|
||||||
|
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
|
||||||
|
for filename in fake_files:
|
||||||
|
fake_file = repo / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
|
fdroidserver.deploy.update_servergitmirrors(
|
||||||
|
fdroidserver.deploy.config["servergitmirrors"], repo_section
|
||||||
)
|
)
|
||||||
|
|
||||||
|
verify_repo = remote_git_repo.clone(
|
||||||
|
Path(self.testdir) / 'verify',
|
||||||
|
)
|
||||||
|
|
||||||
|
for filename in fake_files:
|
||||||
|
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||||
|
|
||||||
|
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||||
|
if verify_repo.working_tree_dir is not None:
|
||||||
|
self.assertTrue(
|
||||||
|
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_update_servergitmirrors_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||||
|
|
||||||
|
remote_repo = Path(self.testdir) / 'remote'
|
||||||
|
remote_repo.mkdir(parents=True)
|
||||||
|
remote_git_repo = git.Repo.init(
|
||||||
|
remote_repo, initial_branch=initial_branch, bare=True
|
||||||
|
)
|
||||||
|
fdroidserver.deploy.config["servergitmirrors"] = [
|
||||||
|
{"url": str(remote_repo), "index_only": True}
|
||||||
|
]
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
repo = Path('repo')
|
||||||
|
repo.mkdir(parents=True)
|
||||||
|
fake_apk = 'Sym.apk'
|
||||||
|
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
|
||||||
|
for filename in fake_files:
|
||||||
|
fake_file = repo / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
|
fdroidserver.deploy.update_servergitmirrors(
|
||||||
|
fdroidserver.deploy.config["servergitmirrors"], repo_section
|
||||||
|
)
|
||||||
|
|
||||||
|
verify_repo = remote_git_repo.clone(
|
||||||
|
Path(self.testdir) / 'verify',
|
||||||
|
)
|
||||||
|
|
||||||
|
for filename in fdroidserver.deploy.INDEX_FILES:
|
||||||
|
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||||
|
|
||||||
|
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||||
|
if verify_repo.working_tree_dir is not None:
|
||||||
|
self.assertTrue(
|
||||||
|
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Should not have the APK file
|
||||||
|
remote_file = f"fdroid/{repo_section}/{fake_apk}"
|
||||||
|
if verify_repo.working_tree_dir is not None:
|
||||||
|
self.assertFalse(
|
||||||
|
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_upload_to_servergitmirror_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options = mock.Mock()
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
initial_branch = fdroidserver.deploy.GIT_BRANCH
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
|
||||||
|
local_git_repo_path = Path(self.testdir) / 'local'
|
||||||
|
local_git_repo = git.Repo.init(
|
||||||
|
local_git_repo_path, initial_branch=initial_branch
|
||||||
|
)
|
||||||
|
|
||||||
|
fdroid_dir = local_git_repo_path / 'fdroid'
|
||||||
|
repo_dir = fdroid_dir / repo_section
|
||||||
|
repo_dir.mkdir(parents=True)
|
||||||
|
fake_apk = 'Sym.apk'
|
||||||
|
fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
|
||||||
|
for filename in fake_files:
|
||||||
|
fake_file = repo_dir / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
|
# The remote repo must be a bare repo to allow being pushed to
|
||||||
|
remote_git_repo_dir = Path(self.testdir) / 'remote'
|
||||||
|
remote_git_repo = git.Repo.init(
|
||||||
|
remote_git_repo_dir, initial_branch=initial_branch, bare=True
|
||||||
|
)
|
||||||
|
|
||||||
|
mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
|
||||||
|
enabled_remotes = []
|
||||||
|
ssh_cmd = 'ssh -oBatchMode=yes'
|
||||||
|
fdroidserver.deploy.upload_to_servergitmirror(
|
||||||
|
mirror_config=mirror_config,
|
||||||
|
local_repo=local_git_repo,
|
||||||
|
enabled_remotes=enabled_remotes,
|
||||||
|
repo_section=repo_section,
|
||||||
|
is_index_only=mirror_config['index_only'],
|
||||||
|
fdroid_dir=str(fdroid_dir),
|
||||||
|
git_mirror_path=str(local_git_repo_path),
|
||||||
|
ssh_cmd=ssh_cmd,
|
||||||
|
progress=git.RemoteProgress(),
|
||||||
|
)
|
||||||
|
|
||||||
|
verify_repo = remote_git_repo.clone(
|
||||||
|
Path(self.testdir) / 'verify',
|
||||||
|
)
|
||||||
|
|
||||||
|
for filename in fdroidserver.deploy.INDEX_FILES:
|
||||||
|
remote_file = f"fdroid/{repo_section}/{filename}"
|
||||||
|
|
||||||
|
self.assertIsNotNone(verify_repo.working_tree_dir)
|
||||||
|
if verify_repo.working_tree_dir is not None:
|
||||||
|
self.assertTrue(
|
||||||
|
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Should not have the APK file
|
||||||
|
remote_file = f"fdroid/{repo_section}/{fake_apk}"
|
||||||
|
if verify_repo.working_tree_dir is not None:
|
||||||
|
self.assertFalse(
|
||||||
|
(Path(verify_repo.working_tree_dir) / remote_file).exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
os.chdir(os.path.dirname(__file__))
|
os.chdir(os.path.dirname(__file__))
|
||||||
|
Loading…
Reference in New Issue
Block a user