From abf4c93a7bdafe4c758f8588164669b95e47a55b Mon Sep 17 00:00:00 2001 From: proletarius101 Date: Mon, 20 May 2024 00:16:27 +0800 Subject: [PATCH] fix(deploy): don't recreate the branches --- fdroidserver/deploy.py | 245 +++++++++++++++++++++-------------------- tests/deploy.TestCase | 48 ++++++++ 2 files changed, 176 insertions(+), 117 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index d2ad5802..d2be4ce5 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -25,12 +25,14 @@ import re import subprocess import time import urllib -from typing import Optional +from typing import Dict, Optional, List +from git import Repo import yaml from argparse import ArgumentParser, Namespace import logging from shlex import split import shutil +import git from . import _ from . import common @@ -621,7 +623,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): transparency log. """ - import git from clint.textui import progress if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): @@ -687,133 +688,143 @@ def update_servergitmirrors(servergitmirrors, repo_section): progress = None repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) + initial_ref = repo.head.ref # An initial commit of the git tree is required be for other operations - initial_branch_ref = repo.head.ref - repo.index.commit('Initial commit') + print("repo.index.entries:", len(repo.index.entries)) + if len(repo.index.entries) == 0: + repo.index.commit('Initial commit') enabled_remotes = [] for d in servergitmirrors: - # Test - print(f"d: {d}") - print("files:") - print(glob.glob('.' + '/**/*', recursive=True)) - - index_only = d.get('index_only', False) - - # Use a separate branch for the index only mode as it needs a different set of files to commit - if index_only: - branch_name = 'index_only' - else: - branch_name = 'full' - if not branch_name in repo.heads: - repo.create_head(branch_name, initial_branch_ref) - repo.head.reference = repo.heads[branch_name] - - # test - print(repo.git.status()) - - remote_url = d['url'] - name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) - enabled_remotes.append(name) - r = git.remote.Remote(repo, name) - if r in repo.remotes: - r = repo.remote(name) - if 'set_url' in dir(r): # force remote URL if using GitPython 2.x - r.set_url(remote_url) - else: - repo.create_remote(name, remote_url) - logging.info('Mirroring to: ' + remote_url) - - if index_only: - # test - print(glob.glob('.' + '/**/*', recursive=True)) - - logging.debug('Adding index files to git mirror') - repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section))) - else: - # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') - repo.git.add(all=True) - - logging.debug('Committing files into git mirror') - repo.index.commit("fdroidserver git-mirror") - - # Test - print(f"In index-only: {index_only} mode") - print(repo.git.status()) - print(repo.head.log()) - - # only deploy to GitLab Artifacts if too big for GitLab Pages - if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: - gitlab_ci_job_name = 'pages' - else: - gitlab_ci_job_name = 'GitLab Artifacts' - logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) - ) - - # push. This will overwrite the git history - remote = repo.remote(name) - if remote.name == 'gitlab': - logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') - with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: - yaml.dump( - { - gitlab_ci_job_name: { - 'script': [ - 'mkdir .public', - 'cp -r * .public/', - 'mv .public public', - ], - 'artifacts': {'paths': ['public']}, - 'variables': {'GIT_DEPTH': 1}, - } - }, - fp, - default_flow_style=False, - ) - - repo.index.add(['.gitlab-ci.yml']) - repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") - - logging.debug(_('Pushing to {url}').format(url=remote.url)) - with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): - pushinfos = remote.push( - GIT_BRANCH, force=True, set_upstream=True, progress=progress - ) - for pushinfo in pushinfos: - if pushinfo.flags & ( - git.remote.PushInfo.ERROR - | git.remote.PushInfo.REJECTED - | git.remote.PushInfo.REMOTE_FAILURE - | git.remote.PushInfo.REMOTE_REJECTED - ): - # Show potentially useful messages from git remote - for line in progress.other_lines: - if line.startswith('remote:'): - logging.debug(line) - raise FDroidException( - '{url} push failed: {flags} {summary}'.format( - url=remote.url, - flags=pushinfo.flags, - summary=pushinfo.summary, - ) - ) - else: - logging.debug(remote.url + ': ' + pushinfo.summary) + upload_to_servergitmirror(mirror_config=d, + local_repo=repo, + enabled_remotes=enabled_remotes, + repo_section=repo_section, + fdroid_dir=git_fdroiddir, + git_mirror_path=git_mirror_path, + ssh_cmd=ssh_cmd) # Switch to the initial branch and unstage all files - repo.head.reference = initial_branch_ref + repo.head.reference = initial_ref repo.head.reset(index=True, working_tree=False) - repo.delete_head(repo.branches[branch_name], force=True) if progress: progressbar.done() +def upload_to_servergitmirror(mirror_config: Dict[str, str], + local_repo: Repo, + enabled_remotes: List[str], + repo_section: str, + fdroid_dir: str, + git_mirror_path: str, + ssh_cmd: str): + # Test + print(f"mirror_config: {mirror_config}") + print("files:") + print(glob.glob('.' + '/**/*', recursive=True)) + + index_only = mirror_config.get('index_only', False) + + # Use a separate branch for the index only mode as it needs a different set of files to commit + if index_only: + branch_name = 'index_only' + else: + branch_name = 'full' + if branch_name not in local_repo.heads: + local_repo.create_head(branch_name) + local_repo.head.reference = local_repo.heads[branch_name] + + # test + print(local_repo.git.status()) + + remote_url = mirror_config['url'] + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(local_repo, name) + if r in local_repo.remotes: + r = local_repo.remote(name) + if 'set_url' in dir(r): # force remote URL if using GitPython 2.x + r.set_url(remote_url) + else: + local_repo.create_remote(name, remote_url) + logging.info('Mirroring to: ' + remote_url) + + if index_only: + # test + print(glob.glob('.' + '/**/*', recursive=True)) + + logging.debug('Adding index files to git mirror') + local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section))) + else: + # sadly index.add don't allow the --all parameter + logging.debug('Adding all files to git mirror') + local_repo.git.add(all=True) + + logging.debug('Committing files into git mirror') + local_repo.index.commit("fdroidserver git-mirror") + + # Test + print(f"In index-only: {index_only} mode") + print(local_repo.git.status()) + print(local_repo.head.log()) + + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + + # push. This will overwrite the git history + remote = local_repo.remote(name) + if remote.name == 'gitlab': + logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') + with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: + yaml.dump( + { + gitlab_ci_job_name: { + 'script': [ + 'mkdir .public', + 'cp -r * .public/', + 'mv .public public', + ], + 'artifacts': {'paths': ['public']}, + 'variables': {'GIT_DEPTH': 1}, + } + }, + fp, + default_flow_style=False, + ) + + local_repo.index.add(['.gitlab-ci.yml']) + local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") + + logging.debug(_('Pushing to {url}').format(url=remote.url)) + with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + pushinfos = remote.push( + GIT_BRANCH, force=True, set_upstream=True, progress=progress + ) + for pushinfo in pushinfos: + if pushinfo.flags & (git.remote.PushInfo.ERROR + | git.remote.PushInfo.REJECTED + | git.remote.PushInfo.REMOTE_FAILURE + | git.remote.PushInfo.REMOTE_REJECTED): + # Show potentially useful messages from git remote + for line in progress.other_lines: + if line.startswith('remote:'): + logging.debug(line) + raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags) + + ' ' + pushinfo.summary) + else: + logging.debug(remote.url + ': ' + pushinfo.summary) + + def upload_to_android_observatory(repo_section): import requests diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index bbc09852..ffd4959d 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -7,10 +7,13 @@ import os import shutil import sys import tempfile +from unicodedata import mirrored import unittest from pathlib import Path from unittest import mock +import git + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') ) @@ -1002,6 +1005,51 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.update_servergitmirrors([], repo_section) self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') + def test_upload_to_servergitmirror_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.deploy.options.identity_file = None + fdroidserver.deploy.options.no_keep_git_mirror_archive = False + fdroidserver.deploy.options.verbose = False + fdroidserver.deploy.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + + repo_section = 'repo' + initial_branch = 'main' + + os.chdir(self.testdir) + + local_git_repo_path = Path(self.testdir) + local_git_repo = git.Repo.init( + local_git_repo_path, initial_branch=initial_branch + ) + fdroid_dir = local_git_repo_path / 'fdroid' + repo_dir = fdroid_dir / repo_section + repo_dir.mkdir(parents=True) + for filename in fdroidserver.deploy.INDEX_FILES: + fake_file = repo_dir / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + remote_git_repo_path = Path('git-mirror') + remote_repo = git.Repo.init(remote_git_repo_path, initial_branch=initial_branch) + + mirror_config = {"url": remote_git_repo_path, "index_only": True} + enabled_remotes = [] + ssh_cmd = 'ssh -oBatchMode=yes' + fdroidserver.deploy.upload_to_servergitmirror( + mirror_config=mirror_config, + local_repo=local_git_repo, + enabled_remotes=enabled_remotes, + repo_section=repo_section, + fdroid_dir=str(fdroid_dir), + git_mirror_path=str(local_git_repo_path), + ssh_cmd=ssh_cmd, + ) + assert remote_repo.is_dirty + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))