mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
Compare commits
6 Commits
40ecaff6fe
...
da5ff1c928
Author | SHA1 | Date | |
---|---|---|---|
|
da5ff1c928 | ||
|
dc3fd0bdd1 | ||
|
c26267a7f9 | ||
|
48c48596a8 | ||
|
5ff9a19072 | ||
|
78b6e53e42 |
@ -16,7 +16,6 @@
|
|||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from math import log
|
|
||||||
import sys
|
import sys
|
||||||
import glob
|
import glob
|
||||||
import hashlib
|
import hashlib
|
||||||
@ -95,7 +94,7 @@ def _get_index_includes(base_dir):
|
|||||||
return index_includes
|
return index_includes
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket(repo_section, index_only=False):
|
def update_awsbucket(repo_section, is_index_only=False):
|
||||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||||
|
|
||||||
The contents of that subdir of the
|
The contents of that subdir of the
|
||||||
@ -107,12 +106,12 @@ def update_awsbucket(repo_section, index_only=False):
|
|||||||
+ config['awsbucket'] + '"')
|
+ config['awsbucket'] + '"')
|
||||||
|
|
||||||
if common.set_command_in_config('s3cmd'):
|
if common.set_command_in_config('s3cmd'):
|
||||||
update_awsbucket_s3cmd(repo_section, index_only)
|
update_awsbucket_s3cmd(repo_section, is_index_only)
|
||||||
else:
|
else:
|
||||||
update_awsbucket_libcloud(repo_section, index_only)
|
update_awsbucket_libcloud(repo_section, is_index_only)
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket_s3cmd(repo_section, index_only=False):
|
def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||||
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
||||||
|
|
||||||
The upload is done in multiple passes to reduce the chance of
|
The upload is done in multiple passes to reduce the chance of
|
||||||
@ -156,7 +155,7 @@ def update_awsbucket_s3cmd(repo_section, index_only=False):
|
|||||||
|
|
||||||
s3url = s3bucketurl + '/fdroid/'
|
s3url = s3bucketurl + '/fdroid/'
|
||||||
|
|
||||||
if index_only:
|
if is_index_only:
|
||||||
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||||
.format(path=repo_section, url=s3url))
|
.format(path=repo_section, url=s3url))
|
||||||
sync_indexes_flags = []
|
sync_indexes_flags = []
|
||||||
@ -200,7 +199,7 @@ def update_awsbucket_s3cmd(repo_section, index_only=False):
|
|||||||
raise FDroidException()
|
raise FDroidException()
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket_libcloud(repo_section, index_only=False):
|
def update_awsbucket_libcloud(repo_section, is_index_only=False):
|
||||||
"""No summary.
|
"""No summary.
|
||||||
|
|
||||||
Upload the contents of the directory `repo_section` (including
|
Upload the contents of the directory `repo_section` (including
|
||||||
@ -243,7 +242,7 @@ def update_awsbucket_libcloud(repo_section, index_only=False):
|
|||||||
if obj.name.startswith(upload_dir + '/'):
|
if obj.name.startswith(upload_dir + '/'):
|
||||||
objs[obj.name] = obj
|
objs[obj.name] = obj
|
||||||
|
|
||||||
if index_only:
|
if is_index_only:
|
||||||
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
||||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||||
files_to_upload = list(set(files_to_upload) & set(index_files))
|
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||||
@ -336,9 +335,9 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||||||
elif config and config.get('identity_file'):
|
elif config and config.get('identity_file'):
|
||||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||||
url = serverwebroot['url']
|
url = serverwebroot['url']
|
||||||
index_only = serverwebroot.get('index_only', False)
|
is_index_only = serverwebroot.get('index_only', False)
|
||||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||||
if index_only:
|
if is_index_only:
|
||||||
rsyncargs += _get_index_file_paths(repo_section)
|
rsyncargs += _get_index_file_paths(repo_section)
|
||||||
rsyncargs += [f'{url}/{repo_section}/']
|
rsyncargs += [f'{url}/{repo_section}/']
|
||||||
logging.info(rsyncargs)
|
logging.info(rsyncargs)
|
||||||
@ -458,11 +457,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
|
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
|
||||||
return
|
return
|
||||||
|
|
||||||
working_dir_root = Path(os.getcwd())
|
workspace_dir = Path(os.getcwd())
|
||||||
|
|
||||||
# right now we support only 'repo' git-mirroring
|
# right now we support only 'repo' git-mirroring
|
||||||
if repo_section == 'repo':
|
if repo_section == 'repo':
|
||||||
git_mirror_path = working_dir_root / 'git-mirror'
|
git_mirror_path = workspace_dir / 'git-mirror'
|
||||||
dotgit = os.path.join(git_mirror_path, '.git')
|
dotgit = os.path.join(git_mirror_path, '.git')
|
||||||
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
|
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
|
||||||
git_repodir = os.path.join(git_fdroiddir, repo_section)
|
git_repodir = os.path.join(git_fdroiddir, repo_section)
|
||||||
@ -520,9 +519,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||||
# make sure both paths have exactly one trailing slash
|
# make sure both paths have exactly one trailing slash
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
|
files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
|
||||||
else:
|
else:
|
||||||
files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
|
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
|
||||||
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
|
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
|
||||||
|
|
||||||
upload_to_servergitmirror(mirror_config=d,
|
upload_to_servergitmirror(mirror_config=d,
|
||||||
@ -534,9 +533,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
git_mirror_path=str(git_mirror_path),
|
git_mirror_path=str(git_mirror_path),
|
||||||
ssh_cmd=ssh_cmd,
|
ssh_cmd=ssh_cmd,
|
||||||
progress=progress)
|
progress=progress)
|
||||||
|
|
||||||
# Switch to the initial branch
|
|
||||||
repo.git.switch(GIT_BRANCH)
|
|
||||||
if progress:
|
if progress:
|
||||||
progressbar.done()
|
progressbar.done()
|
||||||
|
|
||||||
@ -550,12 +546,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
|||||||
git_mirror_path: str,
|
git_mirror_path: str,
|
||||||
ssh_cmd: str,
|
ssh_cmd: str,
|
||||||
progress: git.RemoteProgress) -> None:
|
progress: git.RemoteProgress) -> None:
|
||||||
# Test
|
|
||||||
logging.info(f"mirror_config: {mirror_config}")
|
|
||||||
# List all files in the git repo
|
|
||||||
logging.info("files:")
|
|
||||||
logging.info(local_repo.git.ls_files())
|
|
||||||
|
|
||||||
remote_branch_name = GIT_BRANCH
|
remote_branch_name = GIT_BRANCH
|
||||||
local_branch_name = local_repo.active_branch.name
|
local_branch_name = local_repo.active_branch.name
|
||||||
|
|
||||||
@ -573,8 +563,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
|||||||
|
|
||||||
logging.info("git status:", local_repo.git.status())
|
logging.info("git status:", local_repo.git.status())
|
||||||
if is_index_only:
|
if is_index_only:
|
||||||
# test
|
|
||||||
logging.debug('Adding index files to git mirror')
|
|
||||||
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||||
else:
|
else:
|
||||||
# sadly index.add don't allow the --all parameter
|
# sadly index.add don't allow the --all parameter
|
||||||
@ -584,10 +572,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
|||||||
logging.debug('Committing files into git mirror')
|
logging.debug('Committing files into git mirror')
|
||||||
local_repo.index.commit("fdroidserver git-mirror")
|
local_repo.index.commit("fdroidserver git-mirror")
|
||||||
|
|
||||||
# Test
|
|
||||||
logging.info(f"In index-only: {is_index_only} mode")
|
|
||||||
logging.info(local_repo.git.status())
|
|
||||||
logging.info(local_repo.head.log())
|
|
||||||
|
|
||||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||||
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||||
@ -625,7 +609,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
|||||||
local_repo.index.add(['.gitlab-ci.yml'])
|
local_repo.index.add(['.gitlab-ci.yml'])
|
||||||
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||||
|
|
||||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
logging.info(_('Pushing to {url}').format(url=remote.url))
|
||||||
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||||
pushinfos = remote.push(
|
pushinfos = remote.push(
|
||||||
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
|
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
|
||||||
@ -875,12 +859,6 @@ def main():
|
|||||||
help=_("Don't use rsync checksums"))
|
help=_("Don't use rsync checksums"))
|
||||||
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
|
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
|
||||||
help=_("If a git mirror gets to big, allow the archive to be deleted"))
|
help=_("If a git mirror gets to big, allow the archive to be deleted"))
|
||||||
parser.add_argument(
|
|
||||||
"--index-only",
|
|
||||||
action="store_true",
|
|
||||||
default=False,
|
|
||||||
help="Only deploy the index files entry.* and index-v*.json",
|
|
||||||
)
|
|
||||||
options = parser.parse_args()
|
options = parser.parse_args()
|
||||||
config = common.read_config(options)
|
config = common.read_config(options)
|
||||||
|
|
||||||
|
@ -6,7 +6,6 @@ import optparse
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
from unicodedata import mirrored
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
Loading…
Reference in New Issue
Block a user