mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
Compare commits
No commits in common. "da5ff1c928dc14520d582c1d8d2ff5f6188ca191" and "40ecaff6fe264e74805ee4b09ee188c2ecd7465a" have entirely different histories.
da5ff1c928
...
40ecaff6fe
@ -16,6 +16,7 @@
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from math import log
|
||||
import sys
|
||||
import glob
|
||||
import hashlib
|
||||
@ -94,7 +95,7 @@ def _get_index_includes(base_dir):
|
||||
return index_includes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section, is_index_only=False):
|
||||
def update_awsbucket(repo_section, index_only=False):
|
||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||
|
||||
The contents of that subdir of the
|
||||
@ -106,12 +107,12 @@ def update_awsbucket(repo_section, is_index_only=False):
|
||||
+ config['awsbucket'] + '"')
|
||||
|
||||
if common.set_command_in_config('s3cmd'):
|
||||
update_awsbucket_s3cmd(repo_section, is_index_only)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
else:
|
||||
update_awsbucket_libcloud(repo_section, is_index_only)
|
||||
update_awsbucket_libcloud(repo_section, index_only)
|
||||
|
||||
|
||||
def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||
def update_awsbucket_s3cmd(repo_section, index_only=False):
|
||||
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
||||
|
||||
The upload is done in multiple passes to reduce the chance of
|
||||
@ -155,7 +156,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||
|
||||
s3url = s3bucketurl + '/fdroid/'
|
||||
|
||||
if is_index_only:
|
||||
if index_only:
|
||||
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||
.format(path=repo_section, url=s3url))
|
||||
sync_indexes_flags = []
|
||||
@ -199,7 +200,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_awsbucket_libcloud(repo_section, is_index_only=False):
|
||||
def update_awsbucket_libcloud(repo_section, index_only=False):
|
||||
"""No summary.
|
||||
|
||||
Upload the contents of the directory `repo_section` (including
|
||||
@ -242,7 +243,7 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
|
||||
if obj.name.startswith(upload_dir + '/'):
|
||||
objs[obj.name] = obj
|
||||
|
||||
if is_index_only:
|
||||
if index_only:
|
||||
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||
@ -335,9 +336,9 @@ def update_serverwebroot(serverwebroot, repo_section):
|
||||
elif config and config.get('identity_file'):
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||
url = serverwebroot['url']
|
||||
is_index_only = serverwebroot.get('index_only', False)
|
||||
index_only = serverwebroot.get('index_only', False)
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||
if is_index_only:
|
||||
if index_only:
|
||||
rsyncargs += _get_index_file_paths(repo_section)
|
||||
rsyncargs += [f'{url}/{repo_section}/']
|
||||
logging.info(rsyncargs)
|
||||
@ -457,11 +458,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
|
||||
return
|
||||
|
||||
workspace_dir = Path(os.getcwd())
|
||||
working_dir_root = Path(os.getcwd())
|
||||
|
||||
# right now we support only 'repo' git-mirroring
|
||||
if repo_section == 'repo':
|
||||
git_mirror_path = workspace_dir / 'git-mirror'
|
||||
git_mirror_path = working_dir_root / 'git-mirror'
|
||||
dotgit = os.path.join(git_mirror_path, '.git')
|
||||
git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
|
||||
git_repodir = os.path.join(git_fdroiddir, repo_section)
|
||||
@ -519,9 +520,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
if is_index_only:
|
||||
files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
|
||||
files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
|
||||
else:
|
||||
files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
|
||||
files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
|
||||
common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
|
||||
|
||||
upload_to_servergitmirror(mirror_config=d,
|
||||
@ -533,6 +534,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
||||
git_mirror_path=str(git_mirror_path),
|
||||
ssh_cmd=ssh_cmd,
|
||||
progress=progress)
|
||||
|
||||
# Switch to the initial branch
|
||||
repo.git.switch(GIT_BRANCH)
|
||||
if progress:
|
||||
progressbar.done()
|
||||
|
||||
@ -546,6 +550,12 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||
git_mirror_path: str,
|
||||
ssh_cmd: str,
|
||||
progress: git.RemoteProgress) -> None:
|
||||
# Test
|
||||
logging.info(f"mirror_config: {mirror_config}")
|
||||
# List all files in the git repo
|
||||
logging.info("files:")
|
||||
logging.info(local_repo.git.ls_files())
|
||||
|
||||
remote_branch_name = GIT_BRANCH
|
||||
local_branch_name = local_repo.active_branch.name
|
||||
|
||||
@ -563,6 +573,8 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||
|
||||
logging.info("git status:", local_repo.git.status())
|
||||
if is_index_only:
|
||||
# test
|
||||
logging.debug('Adding index files to git mirror')
|
||||
local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
@ -572,6 +584,10 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||
logging.debug('Committing files into git mirror')
|
||||
local_repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
# Test
|
||||
logging.info(f"In index-only: {is_index_only} mode")
|
||||
logging.info(local_repo.git.status())
|
||||
logging.info(local_repo.head.log())
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
@ -609,7 +625,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
|
||||
local_repo.index.add(['.gitlab-ci.yml'])
|
||||
local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||
|
||||
logging.info(_('Pushing to {url}').format(url=remote.url))
|
||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||
with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
|
||||
pushinfos = remote.push(
|
||||
f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
|
||||
@ -859,6 +875,12 @@ def main():
|
||||
help=_("Don't use rsync checksums"))
|
||||
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
|
||||
help=_("If a git mirror gets to big, allow the archive to be deleted"))
|
||||
parser.add_argument(
|
||||
"--index-only",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Only deploy the index files entry.* and index-v*.json",
|
||||
)
|
||||
options = parser.parse_args()
|
||||
config = common.read_config(options)
|
||||
|
||||
|
@ -6,6 +6,7 @@ import optparse
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from unicodedata import mirrored
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
Loading…
Reference in New Issue
Block a user