Merge branch 'fdroid-mirror' into 'master'

mirror: new command to make a mirror of a repo

See merge request fdroid/fdroidserver!389

completion/bash-completion

@@ -264,6 +264,12 @@ __complete_btlog() {
 	__complete_options
 }
 
+__complete_mirror() {
+	opts="-v"
+	lopts="--archive --output-dir"
+	__complete_options
+}
+
 __complete_nightly() {
 	opts="-v -q"
 	lopts="--show-secret-var"
@@ -316,6 +322,7 @@ import \
 init \
 install \
 lint \
+mirror \
 nightly \
 publish \
 readmeta \

fdroid

@@ -48,6 +48,7 @@ commands = OrderedDict([
     ("btlog", _("Update the binary transparency log for a URL")),
     ("signatures", _("Extract signatures from APKs")),
     ("nightly", _("Set up an app build for a nightly build repo")),
+    ("mirror", _("Download complete mirrors of small repos")),
 ])
 
 

fdroidserver/mirror.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+import ipaddress
+import logging
+import os
+import posixpath
+import socket
+import subprocess
+import sys
+from argparse import ArgumentParser
+import urllib.parse
+
+from . import _
+from . import common
+from . import index
+from . import update
+
+options = None
+
+
+def _run_wget(path, urls):
+    if options.verbose:
+        verbose = '--verbose'
+    else:
+        verbose = '--no-verbose'
+
+    if not urls:
+        return
+    logging.debug(_('Running wget in {path}').format(path=path))
+    os.makedirs(path, exist_ok=True)
+    os.chdir(path)
+    urls_file = '.fdroid-mirror-wget-input-file'
+    with open(urls_file, 'w') as fp:
+        for url in urls:
+            fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
+    subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
+                     '--input-file=' + urls_file])
+    os.remove(urls_file)
+
+
+def main():
+    global options
+
+    parser = ArgumentParser(usage=_("%(prog)s [options] url"))
+    common.setup_global_opts(parser)
+    parser.add_argument("url", nargs='?',
+                        help=_('Base URL to mirror, can include the index signing key '
+                               + 'using the query string: ?fingerprint='))
+    parser.add_argument("--archive", action='store_true', default=False,
+                        help=_("Also mirror the full archive section"))
+    parser.add_argument("--output-dir", default=None,
+                        help=_("The directory to write the mirror to"))
+    options = parser.parse_args()
+
+    if options.url is None:
+        logging.error(_('A URL is required as an argument!') + '\n')
+        parser.print_help()
+        sys.exit(1)
+
+    scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
+    fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
+
+    def _append_to_url_path(*args):
+        '''Append the list of path components to URL, keeping the rest the same'''
+        newpath = posixpath.join(path, *args)
+        return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
+
+    if fingerprint:
+        config = common.read_config(options)
+        if not ('jarsigner' in config or 'apksigner' in config):
+            logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
+            sys.exit(1)
+
+        def _get_index(section, etag=None):
+            url = _append_to_url_path(section)
+            return index.download_repo_index(url, etag=etag)
+    else:
+        def _get_index(section, etag=None):
+            import io
+            import json
+            import zipfile
+            from . import net
+            url = _append_to_url_path(section, 'index-v1.jar')
+            content, etag = net.http_get(url)
+            with zipfile.ZipFile(io.BytesIO(content)) as zip:
+                jsoncontents = zip.open('index-v1.json').read()
+            data = json.loads(jsoncontents.decode('utf-8'))
+            return data, etag
+
+    ip = None
+    try:
+        ip = ipaddress.ip_address(hostname)
+    except ValueError:
+        pass
+    if hostname == 'f-droid.org' \
+       or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
+        print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
+                'A full mirror of f-droid.org requires more than 200GB.'))
+        sys.exit(1)
+
+    path = path.rstrip('/')
+    if path.endswith('repo') or path.endswith('archive'):
+        logging.warning(_('Do not include "{path}" in URL!')
+                        .format(path=path.split('/')[-1]))
+    elif not path.endswith('fdroid'):
+        logging.warning(_('{url} does not end with "fdroid", check the URL path!')
+                        .format(url=options.url))
+
+    icondirs = ['icons', ]
+    for density in update.screen_densities:
+        icondirs.append('icons-' + density)
+
+    if options.output_dir:
+        basedir = options.output_dir
+    else:
+        basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
+        os.makedirs(basedir, exist_ok=True)
+
+    if options.archive:
+        sections = ('repo', 'archive')
+    else:
+        sections = ('repo', )
+
+    for section in sections:
+        sectiondir = os.path.join(basedir, section)
+
+        data, etag = _get_index(section)
+
+        os.makedirs(sectiondir, exist_ok=True)
+        os.chdir(sectiondir)
+        for icondir in icondirs:
+            os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
+
+        urls = []
+        for packageName, packageList in data['packages'].items():
+            for package in packageList:
+                to_fetch = []
+                for k in ('apkName', 'srcname'):
+                    if k in package:
+                        to_fetch.append(package[k])
+                    elif k == 'apkName':
+                        logging.error(_('{appid} is missing {name}')
+                                      .format(appid=package['packageName'], name=k))
+                for f in to_fetch:
+                    if not os.path.exists(f) \
+                       or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
+                        urls.append(_append_to_url_path(section, f))
+                        urls.append(_append_to_url_path(section, f + '.asc'))
+        _run_wget(sectiondir, urls)
+
+        for app in data['apps']:
+            localized = app.get('localized')
+            if localized:
+                for locale, d in localized.items():
+                    urls = []
+                    components = (section, app['packageName'], locale)
+                    for k in update.GRAPHIC_NAMES:
+                        f = d.get(k)
+                        if f:
+                            urls.append(_append_to_url_path(*components, f))
+                    _run_wget(os.path.join(basedir, *components), urls)
+                    for k in update.SCREENSHOT_DIRS:
+                        urls = []
+                        filelist = d.get(k)
+                        if filelist:
+                            components = (section, app['packageName'], locale, k)
+                            for f in filelist:
+                                urls.append(_append_to_url_path(*components, f))
+                            _run_wget(os.path.join(basedir, *components), urls)
+
+        urls = dict()
+        for app in data['apps']:
+            if 'icon' not in app:
+                logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
+                continue
+            icon = app['icon']
+            for icondir in icondirs:
+                url = _append_to_url_path(section, icondir, icon)
+                if icondir not in urls:
+                    urls[icondir] = []
+                urls[icondir].append(url)
+
+        for icondir in icondirs:
+            _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
+
+
+if __name__ == "__main__":
+    main()