1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00

Compare commits

...

6 Commits

Author SHA1 Message Date
proletarius101
da5ff1c928
chore(deploy): rename index_only variable to is_index_only for Python idiomaticness 2024-06-04 20:30:35 +08:00
proletarius101
dc3fd0bdd1
chore(deploy): clean up code 2024-06-04 20:27:01 +08:00
proletarius101
c26267a7f9
fix(deploy): remove the --index-only flag 2024-06-04 20:22:56 +08:00
proletarius101
48c48596a8
fix(deploy): $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk doesn't exist when the test is finished 2024-06-04 20:10:13 +08:00
proletarius101
5ff9a19072
style(deploy): rename working_dir_root variable to workspace_dir 2024-06-05 03:52:02 +08:00
proletarius101
78b6e53e42
style(deploy): fix pyflake 2024-06-05 02:38:14 +08:00
2 changed files with 14 additions and 37 deletions

View File

@ -16,7 +16,6 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from math import log
import sys import sys
import glob import glob
import hashlib import hashlib
@ -95,7 +94,7 @@ def _get_index_includes(base_dir):
return index_includes return index_includes
def update_awsbucket(repo_section, index_only=False): def update_awsbucket(repo_section, is_index_only=False):
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
The contents of that subdir of the The contents of that subdir of the
@ -107,12 +106,12 @@ def update_awsbucket(repo_section, index_only=False):
+ config['awsbucket'] + '"') + config['awsbucket'] + '"')
if common.set_command_in_config('s3cmd'): if common.set_command_in_config('s3cmd'):
update_awsbucket_s3cmd(repo_section, index_only) update_awsbucket_s3cmd(repo_section, is_index_only)
else: else:
update_awsbucket_libcloud(repo_section, index_only) update_awsbucket_libcloud(repo_section, is_index_only)
def update_awsbucket_s3cmd(repo_section, index_only=False): def update_awsbucket_s3cmd(repo_section, is_index_only=False):
"""Upload using the CLI tool s3cmd, which provides rsync-like sync. """Upload using the CLI tool s3cmd, which provides rsync-like sync.
The upload is done in multiple passes to reduce the chance of The upload is done in multiple passes to reduce the chance of
@ -156,7 +155,7 @@ def update_awsbucket_s3cmd(repo_section, index_only=False):
s3url = s3bucketurl + '/fdroid/' s3url = s3bucketurl + '/fdroid/'
if index_only: if is_index_only:
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed') logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
.format(path=repo_section, url=s3url)) .format(path=repo_section, url=s3url))
sync_indexes_flags = [] sync_indexes_flags = []
@ -200,7 +199,7 @@ def update_awsbucket_s3cmd(repo_section, index_only=False):
raise FDroidException() raise FDroidException()
def update_awsbucket_libcloud(repo_section, index_only=False): def update_awsbucket_libcloud(repo_section, is_index_only=False):
"""No summary. """No summary.
Upload the contents of the directory `repo_section` (including Upload the contents of the directory `repo_section` (including
@ -243,7 +242,7 @@ def update_awsbucket_libcloud(repo_section, index_only=False):
if obj.name.startswith(upload_dir + '/'): if obj.name.startswith(upload_dir + '/'):
objs[obj.name] = obj objs[obj.name] = obj
if index_only: if is_index_only:
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)] index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
files_to_upload = list(set(files_to_upload) & set(index_files)) files_to_upload = list(set(files_to_upload) & set(index_files))
@ -336,9 +335,9 @@ def update_serverwebroot(serverwebroot, repo_section):
elif config and config.get('identity_file'): elif config and config.get('identity_file'):
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
url = serverwebroot['url'] url = serverwebroot['url']
index_only = serverwebroot.get('index_only', False) is_index_only = serverwebroot.get('index_only', False)
logging.info('rsyncing ' + repo_section + ' to ' + url) logging.info('rsyncing ' + repo_section + ' to ' + url)
if index_only: if is_index_only:
rsyncargs += _get_index_file_paths(repo_section) rsyncargs += _get_index_file_paths(repo_section)
rsyncargs += [f'{url}/{repo_section}/'] rsyncargs += [f'{url}/{repo_section}/']
logging.info(rsyncargs) logging.info(rsyncargs)
@ -458,11 +457,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`')) logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
return return
working_dir_root = Path(os.getcwd()) workspace_dir = Path(os.getcwd())
# right now we support only 'repo' git-mirroring # right now we support only 'repo' git-mirroring
if repo_section == 'repo': if repo_section == 'repo':
git_mirror_path = working_dir_root / 'git-mirror' git_mirror_path = workspace_dir / 'git-mirror'
dotgit = os.path.join(git_mirror_path, '.git') dotgit = os.path.join(git_mirror_path, '.git')
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid') git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
git_repodir = os.path.join(git_fdroiddir, repo_section) git_repodir = os.path.join(git_fdroiddir, repo_section)
@ -520,9 +519,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
# trailing slashes have a meaning in rsync which is not needed here, so # trailing slashes have a meaning in rsync which is not needed here, so
# make sure both paths have exactly one trailing slash # make sure both paths have exactly one trailing slash
if is_index_only: if is_index_only:
files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES] files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
else: else:
files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/'] files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/') common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
upload_to_servergitmirror(mirror_config=d, upload_to_servergitmirror(mirror_config=d,
@ -534,9 +533,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
git_mirror_path=str(git_mirror_path), git_mirror_path=str(git_mirror_path),
ssh_cmd=ssh_cmd, ssh_cmd=ssh_cmd,
progress=progress) progress=progress)
# Switch to the initial branch
repo.git.switch(GIT_BRANCH)
if progress: if progress:
progressbar.done() progressbar.done()
@ -550,12 +546,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
git_mirror_path: str, git_mirror_path: str,
ssh_cmd: str, ssh_cmd: str,
progress: git.RemoteProgress) -> None: progress: git.RemoteProgress) -> None:
# Test
logging.info(f"mirror_config: {mirror_config}")
# List all files in the git repo
logging.info("files:")
logging.info(local_repo.git.ls_files())
remote_branch_name = GIT_BRANCH remote_branch_name = GIT_BRANCH
local_branch_name = local_repo.active_branch.name local_branch_name = local_repo.active_branch.name
@ -573,8 +563,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
logging.info("git status:", local_repo.git.status()) logging.info("git status:", local_repo.git.status())
if is_index_only: if is_index_only:
# test
logging.debug('Adding index files to git mirror')
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section))) local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
else: else:
# sadly index.add don't allow the --all parameter # sadly index.add don't allow the --all parameter
@ -584,10 +572,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
logging.debug('Committing files into git mirror') logging.debug('Committing files into git mirror')
local_repo.index.commit("fdroidserver git-mirror") local_repo.index.commit("fdroidserver git-mirror")
# Test
logging.info(f"In index-only: {is_index_only} mode")
logging.info(local_repo.git.status())
logging.info(local_repo.head.log())
# only deploy to GitLab Artifacts if too big for GitLab Pages # only deploy to GitLab Artifacts if too big for GitLab Pages
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE: if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
@ -625,7 +609,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
local_repo.index.add(['.gitlab-ci.yml']) local_repo.index.add(['.gitlab-ci.yml'])
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
logging.debug(_('Pushing to {url}').format(url=remote.url)) logging.info(_('Pushing to {url}').format(url=remote.url))
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
pushinfos = remote.push( pushinfos = remote.push(
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
@ -875,12 +859,6 @@ def main():
help=_("Don't use rsync checksums")) help=_("Don't use rsync checksums"))
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False, parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
help=_("If a git mirror gets to big, allow the archive to be deleted")) help=_("If a git mirror gets to big, allow the archive to be deleted"))
parser.add_argument(
"--index-only",
action="store_true",
default=False,
help="Only deploy the index files entry.* and index-v*.json",
)
options = parser.parse_args() options = parser.parse_args()
config = common.read_config(options) config = common.read_config(options)

View File

@ -6,7 +6,6 @@ import optparse
import os import os
import sys import sys
import tempfile import tempfile
from unicodedata import mirrored
import unittest import unittest
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock