mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
fix(deploy): don't recreate the branches
This commit is contained in:
parent
0e781913b8
commit
abf4c93a7b
@ -25,12 +25,14 @@ import re
|
|||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
import urllib
|
import urllib
|
||||||
from typing import Optional
|
from typing import Dict, Optional, List
|
||||||
|
from git import Repo
|
||||||
import yaml
|
import yaml
|
||||||
from argparse import ArgumentParser, Namespace
|
from argparse import ArgumentParser, Namespace
|
||||||
import logging
|
import logging
|
||||||
from shlex import split
|
from shlex import split
|
||||||
import shutil
|
import shutil
|
||||||
|
import git
|
||||||
|
|
||||||
from . import _
|
from . import _
|
||||||
from . import common
|
from . import common
|
||||||
@ -621,7 +623,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
transparency log.
|
transparency log.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import git
|
|
||||||
from clint.textui import progress
|
from clint.textui import progress
|
||||||
|
|
||||||
if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
|
if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
|
||||||
@ -687,133 +688,143 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
progress = None
|
progress = None
|
||||||
|
|
||||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||||
|
initial_ref = repo.head.ref
|
||||||
|
|
||||||
# An initial commit of the git tree is required be for other operations
|
# An initial commit of the git tree is required be for other operations
|
||||||
initial_branch_ref = repo.head.ref
|
print("repo.index.entries:", len(repo.index.entries))
|
||||||
repo.index.commit('Initial commit')
|
if len(repo.index.entries) == 0:
|
||||||
|
repo.index.commit('Initial commit')
|
||||||
|
|
||||||
enabled_remotes = []
|
enabled_remotes = []
|
||||||
for d in servergitmirrors:
|
for d in servergitmirrors:
|
||||||
# Test
|
upload_to_servergitmirror(mirror_config=d,
|
||||||
print(f"d: {d}")
|
local_repo=repo,
|
||||||
print("files:")
|
enabled_remotes=enabled_remotes,
|
||||||
print(glob.glob('.' + '/**/*', recursive=True))
|
repo_section=repo_section,
|
||||||
|
fdroid_dir=git_fdroiddir,
|
||||||
index_only = d.get('index_only', False)
|
git_mirror_path=git_mirror_path,
|
||||||
|
ssh_cmd=ssh_cmd)
|
||||||
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
|
||||||
if index_only:
|
|
||||||
branch_name = 'index_only'
|
|
||||||
else:
|
|
||||||
branch_name = 'full'
|
|
||||||
if not branch_name in repo.heads:
|
|
||||||
repo.create_head(branch_name, initial_branch_ref)
|
|
||||||
repo.head.reference = repo.heads[branch_name]
|
|
||||||
|
|
||||||
# test
|
|
||||||
print(repo.git.status())
|
|
||||||
|
|
||||||
remote_url = d['url']
|
|
||||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
|
||||||
enabled_remotes.append(name)
|
|
||||||
r = git.remote.Remote(repo, name)
|
|
||||||
if r in repo.remotes:
|
|
||||||
r = repo.remote(name)
|
|
||||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
|
||||||
r.set_url(remote_url)
|
|
||||||
else:
|
|
||||||
repo.create_remote(name, remote_url)
|
|
||||||
logging.info('Mirroring to: ' + remote_url)
|
|
||||||
|
|
||||||
if index_only:
|
|
||||||
# test
|
|
||||||
print(glob.glob('.' + '/**/*', recursive=True))
|
|
||||||
|
|
||||||
logging.debug('Adding index files to git mirror')
|
|
||||||
repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
|
||||||
else:
|
|
||||||
# sadly index.add don't allow the --all parameter
|
|
||||||
logging.debug('Adding all files to git mirror')
|
|
||||||
repo.git.add(all=True)
|
|
||||||
|
|
||||||
logging.debug('Committing files into git mirror')
|
|
||||||
repo.index.commit("fdroidserver git-mirror")
|
|
||||||
|
|
||||||
# Test
|
|
||||||
print(f"In index-only: {index_only} mode")
|
|
||||||
print(repo.git.status())
|
|
||||||
print(repo.head.log())
|
|
||||||
|
|
||||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
|
||||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
|
||||||
gitlab_ci_job_name = 'pages'
|
|
||||||
else:
|
|
||||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
|
||||||
logging.warning(
|
|
||||||
_(
|
|
||||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
|
||||||
)
|
|
||||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
|
||||||
)
|
|
||||||
|
|
||||||
# push. This will overwrite the git history
|
|
||||||
remote = repo.remote(name)
|
|
||||||
if remote.name == 'gitlab':
|
|
||||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
|
||||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
|
||||||
yaml.dump(
|
|
||||||
{
|
|
||||||
gitlab_ci_job_name: {
|
|
||||||
'script': [
|
|
||||||
'mkdir .public',
|
|
||||||
'cp -r * .public/',
|
|
||||||
'mv .public public',
|
|
||||||
],
|
|
||||||
'artifacts': {'paths': ['public']},
|
|
||||||
'variables': {'GIT_DEPTH': 1},
|
|
||||||
}
|
|
||||||
},
|
|
||||||
fp,
|
|
||||||
default_flow_style=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
repo.index.add(['.gitlab-ci.yml'])
|
|
||||||
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
|
||||||
|
|
||||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
|
||||||
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
|
||||||
pushinfos = remote.push(
|
|
||||||
GIT_BRANCH, force=True, set_upstream=True, progress=progress
|
|
||||||
)
|
|
||||||
for pushinfo in pushinfos:
|
|
||||||
if pushinfo.flags & (
|
|
||||||
git.remote.PushInfo.ERROR
|
|
||||||
| git.remote.PushInfo.REJECTED
|
|
||||||
| git.remote.PushInfo.REMOTE_FAILURE
|
|
||||||
| git.remote.PushInfo.REMOTE_REJECTED
|
|
||||||
):
|
|
||||||
# Show potentially useful messages from git remote
|
|
||||||
for line in progress.other_lines:
|
|
||||||
if line.startswith('remote:'):
|
|
||||||
logging.debug(line)
|
|
||||||
raise FDroidException(
|
|
||||||
'{url} push failed: {flags} {summary}'.format(
|
|
||||||
url=remote.url,
|
|
||||||
flags=pushinfo.flags,
|
|
||||||
summary=pushinfo.summary,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
|
||||||
|
|
||||||
# Switch to the initial branch and unstage all files
|
# Switch to the initial branch and unstage all files
|
||||||
repo.head.reference = initial_branch_ref
|
repo.head.reference = initial_ref
|
||||||
repo.head.reset(index=True, working_tree=False)
|
repo.head.reset(index=True, working_tree=False)
|
||||||
repo.delete_head(repo.branches[branch_name], force=True)
|
|
||||||
if progress:
|
if progress:
|
||||||
progressbar.done()
|
progressbar.done()
|
||||||
|
|
||||||
|
|
||||||
|
def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||||
|
local_repo: Repo,
|
||||||
|
enabled_remotes: List[str],
|
||||||
|
repo_section: str,
|
||||||
|
fdroid_dir: str,
|
||||||
|
git_mirror_path: str,
|
||||||
|
ssh_cmd: str):
|
||||||
|
# Test
|
||||||
|
print(f"mirror_config: {mirror_config}")
|
||||||
|
print("files:")
|
||||||
|
print(glob.glob('.' + '/**/*', recursive=True))
|
||||||
|
|
||||||
|
index_only = mirror_config.get('index_only', False)
|
||||||
|
|
||||||
|
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
||||||
|
if index_only:
|
||||||
|
branch_name = 'index_only'
|
||||||
|
else:
|
||||||
|
branch_name = 'full'
|
||||||
|
if branch_name not in local_repo.heads:
|
||||||
|
local_repo.create_head(branch_name)
|
||||||
|
local_repo.head.reference = local_repo.heads[branch_name]
|
||||||
|
|
||||||
|
# test
|
||||||
|
print(local_repo.git.status())
|
||||||
|
|
||||||
|
remote_url = mirror_config['url']
|
||||||
|
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||||
|
enabled_remotes.append(name)
|
||||||
|
r = git.remote.Remote(local_repo, name)
|
||||||
|
if r in local_repo.remotes:
|
||||||
|
r = local_repo.remote(name)
|
||||||
|
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||||
|
r.set_url(remote_url)
|
||||||
|
else:
|
||||||
|
local_repo.create_remote(name, remote_url)
|
||||||
|
logging.info('Mirroring to: ' + remote_url)
|
||||||
|
|
||||||
|
if index_only:
|
||||||
|
# test
|
||||||
|
print(glob.glob('.' + '/**/*', recursive=True))
|
||||||
|
|
||||||
|
logging.debug('Adding index files to git mirror')
|
||||||
|
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||||
|
else:
|
||||||
|
# sadly index.add don't allow the --all parameter
|
||||||
|
logging.debug('Adding all files to git mirror')
|
||||||
|
local_repo.git.add(all=True)
|
||||||
|
|
||||||
|
logging.debug('Committing files into git mirror')
|
||||||
|
local_repo.index.commit("fdroidserver git-mirror")
|
||||||
|
|
||||||
|
# Test
|
||||||
|
print(f"In index-only: {index_only} mode")
|
||||||
|
print(local_repo.git.status())
|
||||||
|
print(local_repo.head.log())
|
||||||
|
|
||||||
|
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||||
|
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||||
|
gitlab_ci_job_name = 'pages'
|
||||||
|
else:
|
||||||
|
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||||
|
logging.warning(
|
||||||
|
_(
|
||||||
|
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||||
|
)
|
||||||
|
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||||
|
)
|
||||||
|
|
||||||
|
# push. This will overwrite the git history
|
||||||
|
remote = local_repo.remote(name)
|
||||||
|
if remote.name == 'gitlab':
|
||||||
|
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||||
|
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||||
|
yaml.dump(
|
||||||
|
{
|
||||||
|
gitlab_ci_job_name: {
|
||||||
|
'script': [
|
||||||
|
'mkdir .public',
|
||||||
|
'cp -r * .public/',
|
||||||
|
'mv .public public',
|
||||||
|
],
|
||||||
|
'artifacts': {'paths': ['public']},
|
||||||
|
'variables': {'GIT_DEPTH': 1},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
fp,
|
||||||
|
default_flow_style=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
local_repo.index.add(['.gitlab-ci.yml'])
|
||||||
|
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||||
|
|
||||||
|
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||||
|
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||||
|
pushinfos = remote.push(
|
||||||
|
GIT_BRANCH, force=True, set_upstream=True, progress=progress
|
||||||
|
)
|
||||||
|
for pushinfo in pushinfos:
|
||||||
|
if pushinfo.flags & (git.remote.PushInfo.ERROR
|
||||||
|
| git.remote.PushInfo.REJECTED
|
||||||
|
| git.remote.PushInfo.REMOTE_FAILURE
|
||||||
|
| git.remote.PushInfo.REMOTE_REJECTED):
|
||||||
|
# Show potentially useful messages from git remote
|
||||||
|
for line in progress.other_lines:
|
||||||
|
if line.startswith('remote:'):
|
||||||
|
logging.debug(line)
|
||||||
|
raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
|
||||||
|
+ ' ' + pushinfo.summary)
|
||||||
|
else:
|
||||||
|
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||||
|
|
||||||
|
|
||||||
def upload_to_android_observatory(repo_section):
|
def upload_to_android_observatory(repo_section):
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
@ -7,10 +7,13 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from unicodedata import mirrored
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
import git
|
||||||
|
|
||||||
localmodule = os.path.realpath(
|
localmodule = os.path.realpath(
|
||||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||||
)
|
)
|
||||||
@ -1002,6 +1005,51 @@ class DeployTest(unittest.TestCase):
|
|||||||
fdroidserver.deploy.update_servergitmirrors([], repo_section)
|
fdroidserver.deploy.update_servergitmirrors([], repo_section)
|
||||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||||
|
|
||||||
|
def test_upload_to_servergitmirror_in_index_only_mode(self):
|
||||||
|
# setup parameters for this test run
|
||||||
|
fdroidserver.deploy.options.identity_file = None
|
||||||
|
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||||
|
fdroidserver.deploy.options.verbose = False
|
||||||
|
fdroidserver.deploy.options.quiet = True
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.deploy.config = config
|
||||||
|
|
||||||
|
repo_section = 'repo'
|
||||||
|
initial_branch = 'main'
|
||||||
|
|
||||||
|
os.chdir(self.testdir)
|
||||||
|
|
||||||
|
local_git_repo_path = Path(self.testdir)
|
||||||
|
local_git_repo = git.Repo.init(
|
||||||
|
local_git_repo_path, initial_branch=initial_branch
|
||||||
|
)
|
||||||
|
fdroid_dir = local_git_repo_path / 'fdroid'
|
||||||
|
repo_dir = fdroid_dir / repo_section
|
||||||
|
repo_dir.mkdir(parents=True)
|
||||||
|
for filename in fdroidserver.deploy.INDEX_FILES:
|
||||||
|
fake_file = repo_dir / filename
|
||||||
|
with fake_file.open('w') as fp:
|
||||||
|
fp.write('not a real one, but has the right filename')
|
||||||
|
|
||||||
|
remote_git_repo_path = Path('git-mirror')
|
||||||
|
remote_repo = git.Repo.init(remote_git_repo_path, initial_branch=initial_branch)
|
||||||
|
|
||||||
|
mirror_config = {"url": remote_git_repo_path, "index_only": True}
|
||||||
|
enabled_remotes = []
|
||||||
|
ssh_cmd = 'ssh -oBatchMode=yes'
|
||||||
|
fdroidserver.deploy.upload_to_servergitmirror(
|
||||||
|
mirror_config=mirror_config,
|
||||||
|
local_repo=local_git_repo,
|
||||||
|
enabled_remotes=enabled_remotes,
|
||||||
|
repo_section=repo_section,
|
||||||
|
fdroid_dir=str(fdroid_dir),
|
||||||
|
git_mirror_path=str(local_git_repo_path),
|
||||||
|
ssh_cmd=ssh_cmd,
|
||||||
|
)
|
||||||
|
assert remote_repo.is_dirty
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
os.chdir(os.path.dirname(__file__))
|
os.chdir(os.path.dirname(__file__))
|
||||||
|
Loading…
Reference in New Issue
Block a user