Merge branch 'feature/git-mirroring' into 'master'

Add git repo mirroring

Closes #235

See merge request !206

examples/config.py

@@ -155,8 +155,20 @@ The repository of older versions of applications from the main demo repository.
 #     'bar.info:/var/www/fdroid',
 #     }
 
+# The full URL to a git remote repository. You can include
+# multiple servers to mirror to by wrapping the whole thing in {} or [], and
+# including the servergitmirrors strings in a comma-separated list.
+# Servers listed here will also be automatically inserted in the mirrors list.
+#
+# servergitmirrors = 'https://github.com/user/repo'
+# servergitmirrors = {
+#     'https://github.com/user/repo',
+#     'https://gitlab.com/user/repo',
+#     }
+
 # Any mirrors of this repo, for example all of the servers declared in
-# serverwebroot, will automatically be used by the client.  If one
+# serverwebroot and all the servers declared in servergitmirrors,
+# will automatically be used by the client.  If one
 # mirror is not working, then the client will try another.  If the
 # client has Tor enabled, then the client will prefer mirrors with
 # .onion addresses. This base URL will be used for both the main repo

fdroidserver/common.py

@@ -262,6 +262,15 @@ def read_config(opts, config_file='config.py'):
             rootlist.append(rootstr.replace('//', '/'))
         config['serverwebroot'] = rootlist
 
+    if 'servergitmirrors' in config:
+        if isinstance(config['servergitmirrors'], str):
+            roots = [config['servergitmirrors']]
+        elif all(isinstance(item, str) for item in config['servergitmirrors']):
+            roots = config['servergitmirrors']
+        else:
+            raise TypeError('only accepts strings, lists, and tuples')
+        config['servergitmirrors'] = roots
+
     return config
 
 

fdroidserver/server.py

@@ -25,6 +25,7 @@ import pwd
 import subprocess
 from argparse import ArgumentParser
 import logging
+import shutil
 
 from . import common
 
@@ -191,6 +192,39 @@ def update_localcopy(repo_section, local_copy_dir):
     _local_sync(repo_section, local_copy_dir)
 
 
+def update_servergitmirrors(servergitmirrors, repo_section):
+    # depend on GitPython only if users set a git mirror
+    import git
+    # right now we support only 'repo' git-mirroring
+    if repo_section == 'repo':
+        # create a new git-mirror folder
+        repo_dir = os.path.join('.', 'git-mirror/')
+
+        # remove if already present
+        if os.path.isdir(repo_dir):
+            shutil.rmtree(repo_dir)
+
+        repo = git.Repo.init(repo_dir)
+
+        # take care of each mirror
+        for mirror in servergitmirrors:
+            hostname = mirror.split("/")[2]
+            repo.create_remote(hostname, mirror)
+            logging.info('Mirroring to: ' + mirror)
+
+        # copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync
+        fdroid_repo_path = os.path.join(repo_dir, "fdroid")
+        _local_sync(repo_section, fdroid_repo_path)
+
+        # sadly index.add don't allow the --all parameter
+        repo.git.add(all=True)
+        repo.index.commit("fdroidserver git-mirror")
+
+        # push for every remote. This will overwrite the git history
+        for remote in repo.remotes:
+            remote.push('master', force=True, set_upstream=True)
+
+
 def main():
     global config, options
 
@@ -269,8 +303,9 @@ def main():
 
     if not config.get('awsbucket') \
             and not config.get('serverwebroot') \
+            and not config.get('servergitmirrors') \
             and local_copy_dir is None:
-        logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!'
+        logging.warn('No serverwebroot, local_copy_dir, or awsbucket set! '
                      + 'Edit your config.py to set at least one.')
         sys.exit(1)
 
@@ -313,6 +348,10 @@ def main():
                     update_localcopy(repo_section, local_copy_dir)
             for serverwebroot in config.get('serverwebroot', []):
                 update_serverwebroot(serverwebroot, repo_section)
+            if config.get('servergitmirrors', []):
+                # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
+                servergitmirrors = config.get('servergitmirrors', [])
+                update_servergitmirrors(servergitmirrors, repo_section)
             if config.get('awsbucket'):
                 update_awsbucket(repo_section)
 

fdroidserver/update.py

@@ -961,6 +961,32 @@ def extract_pubkey():
     return hexlify(pubkey)
 
 
+# Get raw URL from git service for mirroring
+def get_raw_mirror(url):
+    # Divide urls in parts
+    url = url.split("/")
+
+    # Get the hostname
+    hostname = url[2]
+
+    # fdroidserver will use always 'master' branch for git-mirroring
+    branch = "master"
+    folder = "fdroid"
+
+    if hostname == "github.com":
+        # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
+        url[2] = "raw.githubusercontent.com"
+        url.extend([branch, folder])
+    elif hostname == "gitlab.com":
+        # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
+        url.extend(["raw", branch, folder])
+    else:
+        return None
+
+    url = "/".join(url)
+    return url
+
+
 def make_index(apps, sortedids, apks, repodir, archive):
     """Generate the repo index files.
 
@@ -1012,6 +1038,10 @@ def make_index(apps, sortedids, apks, repodir, archive):
             mirrors.append(mirror)
         else:
             mirrors.append(mirror + '/')
+    for mirror in config.get('servergitmirrors', []):
+        mirror = get_raw_mirror(mirror)
+        if mirror is not None:
+            mirrors.append(mirror + '/')
     if mirrorcheckfailed:
         sys.exit(1)