mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
fix(deploy): don't recreate the branches
This commit is contained in:
proletarius101
parent
0e781913b8
commit
abf4c93a7b
@ -25,12 +25,14 @@ import re
|
||||
import subprocess
|
||||
import time
|
||||
import urllib
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional, List
|
||||
from git import Repo
|
||||
import yaml
|
||||
from argparse import ArgumentParser, Namespace
|
||||
import logging
|
||||
from shlex import split
|
||||
import shutil
|
||||
import git
|
||||
|
||||
from . import _
|
||||
from . import common
|
||||
@ -621,7 +623,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
transparency log.
|
||||
|
||||
"""
|
||||
import git
|
||||
from clint.textui import progress
|
||||
|
||||
if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
|
||||
@ -687,133 +688,143 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
progress = None
|
||||
|
||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||
initial_ref = repo.head.ref
|
||||
|
||||
# An initial commit of the git tree is required be for other operations
|
||||
initial_branch_ref = repo.head.ref
|
||||
repo.index.commit('Initial commit')
|
||||
print("repo.index.entries:", len(repo.index.entries))
|
||||
if len(repo.index.entries) == 0:
|
||||
repo.index.commit('Initial commit')
|
||||
|
||||
enabled_remotes = []
|
||||
for d in servergitmirrors:
|
||||
# Test
|
||||
print(f"d: {d}")
|
||||
print("files:")
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
index_only = d.get('index_only', False)
|
||||
|
||||
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
||||
if index_only:
|
||||
branch_name = 'index_only'
|
||||
else:
|
||||
branch_name = 'full'
|
||||
if not branch_name in repo.heads:
|
||||
repo.create_head(branch_name, initial_branch_ref)
|
||||
repo.head.reference = repo.heads[branch_name]
|
||||
|
||||
# test
|
||||
print(repo.git.status())
|
||||
|
||||
remote_url = d['url']
|
||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
enabled_remotes.append(name)
|
||||
r = git.remote.Remote(repo, name)
|
||||
if r in repo.remotes:
|
||||
r = repo.remote(name)
|
||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||
r.set_url(remote_url)
|
||||
else:
|
||||
repo.create_remote(name, remote_url)
|
||||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
if index_only:
|
||||
# test
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
logging.debug('Adding index files to git mirror')
|
||||
repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
repo.git.add(all=True)
|
||||
|
||||
logging.debug('Committing files into git mirror')
|
||||
repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
# Test
|
||||
print(f"In index-only: {index_only} mode")
|
||||
print(repo.git.status())
|
||||
print(repo.head.log())
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
gitlab_ci_job_name = 'pages'
|
||||
else:
|
||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||
logging.warning(
|
||||
_(
|
||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||
)
|
||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||
)
|
||||
|
||||
# push. This will overwrite the git history
|
||||
remote = repo.remote(name)
|
||||
if remote.name == 'gitlab':
|
||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||
yaml.dump(
|
||||
{
|
||||
gitlab_ci_job_name: {
|
||||
'script': [
|
||||
'mkdir .public',
|
||||
'cp -r * .public/',
|
||||
'mv .public public',
|
||||
],
|
||||
'artifacts': {'paths': ['public']},
|
||||
'variables': {'GIT_DEPTH': 1},
|
||||
}
|
||||
},
|
||||
fp,
|
||||
default_flow_style=False,
|
||||
)
|
||||
|
||||
repo.index.add(['.gitlab-ci.yml'])
|
||||
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||
|
||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||
pushinfos = remote.push(
|
||||
GIT_BRANCH, force=True, set_upstream=True, progress=progress
|
||||
)
|
||||
for pushinfo in pushinfos:
|
||||
if pushinfo.flags & (
|
||||
git.remote.PushInfo.ERROR
|
||||
| git.remote.PushInfo.REJECTED
|
||||
| git.remote.PushInfo.REMOTE_FAILURE
|
||||
| git.remote.PushInfo.REMOTE_REJECTED
|
||||
):
|
||||
# Show potentially useful messages from git remote
|
||||
for line in progress.other_lines:
|
||||
if line.startswith('remote:'):
|
||||
logging.debug(line)
|
||||
raise FDroidException(
|
||||
'{url} push failed: {flags} {summary}'.format(
|
||||
url=remote.url,
|
||||
flags=pushinfo.flags,
|
||||
summary=pushinfo.summary,
|
||||
)
|
||||
)
|
||||
else:
|
||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||
upload_to_servergitmirror(mirror_config=d,
|
||||
local_repo=repo,
|
||||
enabled_remotes=enabled_remotes,
|
||||
repo_section=repo_section,
|
||||
fdroid_dir=git_fdroiddir,
|
||||
git_mirror_path=git_mirror_path,
|
||||
ssh_cmd=ssh_cmd)
|
||||
|
||||
# Switch to the initial branch and unstage all files
|
||||
repo.head.reference = initial_branch_ref
|
||||
repo.head.reference = initial_ref
|
||||
repo.head.reset(index=True, working_tree=False)
|
||||
repo.delete_head(repo.branches[branch_name], force=True)
|
||||
if progress:
|
||||
progressbar.done()
|
||||
|
||||
|
||||
def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||
local_repo: Repo,
|
||||
enabled_remotes: List[str],
|
||||
repo_section: str,
|
||||
fdroid_dir: str,
|
||||
git_mirror_path: str,
|
||||
ssh_cmd: str):
|
||||
# Test
|
||||
print(f"mirror_config: {mirror_config}")
|
||||
print("files:")
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
index_only = mirror_config.get('index_only', False)
|
||||
|
||||
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
||||
if index_only:
|
||||
branch_name = 'index_only'
|
||||
else:
|
||||
branch_name = 'full'
|
||||
if branch_name not in local_repo.heads:
|
||||
local_repo.create_head(branch_name)
|
||||
local_repo.head.reference = local_repo.heads[branch_name]
|
||||
|
||||
# test
|
||||
print(local_repo.git.status())
|
||||
|
||||
remote_url = mirror_config['url']
|
||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
enabled_remotes.append(name)
|
||||
r = git.remote.Remote(local_repo, name)
|
||||
if r in local_repo.remotes:
|
||||
r = local_repo.remote(name)
|
||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||
r.set_url(remote_url)
|
||||
else:
|
||||
local_repo.create_remote(name, remote_url)
|
||||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
if index_only:
|
||||
# test
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
logging.debug('Adding index files to git mirror')
|
||||
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
local_repo.git.add(all=True)
|
||||
|
||||
logging.debug('Committing files into git mirror')
|
||||
local_repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
# Test
|
||||
print(f"In index-only: {index_only} mode")
|
||||
print(local_repo.git.status())
|
||||
print(local_repo.head.log())
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
gitlab_ci_job_name = 'pages'
|
||||
else:
|
||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||
logging.warning(
|
||||
_(
|
||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||
)
|
||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||
)
|
||||
|
||||
# push. This will overwrite the git history
|
||||
remote = local_repo.remote(name)
|
||||
if remote.name == 'gitlab':
|
||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||
yaml.dump(
|
||||
{
|
||||
gitlab_ci_job_name: {
|
||||
'script': [
|
||||
'mkdir .public',
|
||||
'cp -r * .public/',
|
||||
'mv .public public',
|
||||
],
|
||||
'artifacts': {'paths': ['public']},
|
||||
'variables': {'GIT_DEPTH': 1},
|
||||
}
|
||||
},
|
||||
fp,
|
||||
default_flow_style=False,
|
||||
)
|
||||
|
||||
local_repo.index.add(['.gitlab-ci.yml'])
|
||||
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||
|
||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||
pushinfos = remote.push(
|
||||
GIT_BRANCH, force=True, set_upstream=True, progress=progress
|
||||
)
|
||||
for pushinfo in pushinfos:
|
||||
if pushinfo.flags & (git.remote.PushInfo.ERROR
|
||||
| git.remote.PushInfo.REJECTED
|
||||
| git.remote.PushInfo.REMOTE_FAILURE
|
||||
| git.remote.PushInfo.REMOTE_REJECTED):
|
||||
# Show potentially useful messages from git remote
|
||||
for line in progress.other_lines:
|
||||
if line.startswith('remote:'):
|
||||
logging.debug(line)
|
||||
raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
|
||||
+ ' ' + pushinfo.summary)
|
||||
else:
|
||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||
|
||||
|
||||
def upload_to_android_observatory(repo_section):
|
||||
import requests
|
||||
|
||||
|
||||
@ -7,10 +7,13 @@ import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from unicodedata import mirrored
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import git
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||
)
|
||||
@ -1002,6 +1005,51 @@ class DeployTest(unittest.TestCase):
|
||||
fdroidserver.deploy.update_servergitmirrors([], repo_section)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
def test_upload_to_servergitmirror_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
|
||||
repo_section = 'repo'
|
||||
initial_branch = 'main'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
|
||||
local_git_repo_path = Path(self.testdir)
|
||||
local_git_repo = git.Repo.init(
|
||||
local_git_repo_path, initial_branch=initial_branch
|
||||
)
|
||||
fdroid_dir = local_git_repo_path / 'fdroid'
|
||||
repo_dir = fdroid_dir / repo_section
|
||||
repo_dir.mkdir(parents=True)
|
||||
for filename in fdroidserver.deploy.INDEX_FILES:
|
||||
fake_file = repo_dir / filename
|
||||
with fake_file.open('w') as fp:
|
||||
fp.write('not a real one, but has the right filename')
|
||||
|
||||
remote_git_repo_path = Path('git-mirror')
|
||||
remote_repo = git.Repo.init(remote_git_repo_path, initial_branch=initial_branch)
|
||||
|
||||
mirror_config = {"url": remote_git_repo_path, "index_only": True}
|
||||
enabled_remotes = []
|
||||
ssh_cmd = 'ssh -oBatchMode=yes'
|
||||
fdroidserver.deploy.upload_to_servergitmirror(
|
||||
mirror_config=mirror_config,
|
||||
local_repo=local_git_repo,
|
||||
enabled_remotes=enabled_remotes,
|
||||
repo_section=repo_section,
|
||||
fdroid_dir=str(fdroid_dir),
|
||||
git_mirror_path=str(local_git_repo_path),
|
||||
ssh_cmd=ssh_cmd,
|
||||
)
|
||||
assert remote_repo.is_dirty
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user