diff --git a/examples/config.py b/examples/config.py index 176e576c..a047a71d 100644 --- a/examples/config.py +++ b/examples/config.py @@ -155,8 +155,20 @@ The repository of older versions of applications from the main demo repository. # 'bar.info:/var/www/fdroid', # } +# The full URL to a git remote repository. You can include +# multiple servers to mirror to by wrapping the whole thing in {} or [], and +# including the servergitmirrors strings in a comma-separated list. +# Servers listed here will also be automatically inserted in the mirrors list. +# +# servergitmirrors = 'https://github.com/user/repo' +# servergitmirrors = { +# 'https://github.com/user/repo', +# 'https://gitlab.com/user/repo', +# } + # Any mirrors of this repo, for example all of the servers declared in -# serverwebroot, will automatically be used by the client. If one +# serverwebroot and all the servers declared in servergitmirrors, +# will automatically be used by the client. If one # mirror is not working, then the client will try another. If the # client has Tor enabled, then the client will prefer mirrors with # .onion addresses. This base URL will be used for both the main repo diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 2c8e3630..daacc89d 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -262,6 +262,15 @@ def read_config(opts, config_file='config.py'): rootlist.append(rootstr.replace('//', '/')) config['serverwebroot'] = rootlist + if 'servergitmirrors' in config: + if isinstance(config['servergitmirrors'], str): + roots = [config['servergitmirrors']] + elif all(isinstance(item, str) for item in config['servergitmirrors']): + roots = config['servergitmirrors'] + else: + raise TypeError('only accepts strings, lists, and tuples') + config['servergitmirrors'] = roots + return config diff --git a/fdroidserver/server.py b/fdroidserver/server.py index e40bb55c..6a63b41b 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -25,6 +25,7 @@ import pwd import subprocess from argparse import ArgumentParser import logging +import shutil from . import common @@ -191,6 +192,39 @@ def update_localcopy(repo_section, local_copy_dir): _local_sync(repo_section, local_copy_dir) +def update_servergitmirrors(servergitmirrors, repo_section): + # depend on GitPython only if users set a git mirror + import git + # right now we support only 'repo' git-mirroring + if repo_section == 'repo': + # create a new git-mirror folder + repo_dir = os.path.join('.', 'git-mirror/') + + # remove if already present + if os.path.isdir(repo_dir): + shutil.rmtree(repo_dir) + + repo = git.Repo.init(repo_dir) + + # take care of each mirror + for mirror in servergitmirrors: + hostname = mirror.split("/")[2] + repo.create_remote(hostname, mirror) + logging.info('Mirroring to: ' + mirror) + + # copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync + fdroid_repo_path = os.path.join(repo_dir, "fdroid") + _local_sync(repo_section, fdroid_repo_path) + + # sadly index.add don't allow the --all parameter + repo.git.add(all=True) + repo.index.commit("fdroidserver git-mirror") + + # push for every remote. This will overwrite the git history + for remote in repo.remotes: + remote.push('master', force=True, set_upstream=True) + + def main(): global config, options @@ -269,8 +303,9 @@ def main(): if not config.get('awsbucket') \ and not config.get('serverwebroot') \ + and not config.get('servergitmirrors') \ and local_copy_dir is None: - logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!' + logging.warn('No serverwebroot, local_copy_dir, or awsbucket set! ' + 'Edit your config.py to set at least one.') sys.exit(1) @@ -313,6 +348,10 @@ def main(): update_localcopy(repo_section, local_copy_dir) for serverwebroot in config.get('serverwebroot', []): update_serverwebroot(serverwebroot, repo_section) + if config.get('servergitmirrors', []): + # update_servergitmirrors will take care of multiple mirrors so don't need a foreach + servergitmirrors = config.get('servergitmirrors', []) + update_servergitmirrors(servergitmirrors, repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 48509e65..8129590b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -961,6 +961,32 @@ def extract_pubkey(): return hexlify(pubkey) +# Get raw URL from git service for mirroring +def get_raw_mirror(url): + # Divide urls in parts + url = url.split("/") + + # Get the hostname + hostname = url[2] + + # fdroidserver will use always 'master' branch for git-mirroring + branch = "master" + folder = "fdroid" + + if hostname == "github.com": + # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid" + url[2] = "raw.githubusercontent.com" + url.extend([branch, folder]) + elif hostname == "gitlab.com": + # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid" + url.extend(["raw", branch, folder]) + else: + return None + + url = "/".join(url) + return url + + def make_index(apps, sortedids, apks, repodir, archive): """Generate the repo index files. @@ -1012,6 +1038,10 @@ def make_index(apps, sortedids, apks, repodir, archive): mirrors.append(mirror) else: mirrors.append(mirror + '/') + for mirror in config.get('servergitmirrors', []): + mirror = get_raw_mirror(mirror) + if mirror is not None: + mirrors.append(mirror + '/') if mirrorcheckfailed: sys.exit(1)