1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-05 15:00:11 +01:00
fdroidserver/fdroid
Hans-Christoph Steiner 2162703a1a mirror: new command to make a mirror of a repo
This creates a mirror of a full repo by downloading all files listed in
the index, and the ones that are generated based on that data, e.g. icons
of different resolutions.  This could be useful for setting up mirrors of
small repositories, instead of having to learn and manage rsync or
something else for mirroring.  This just needs a working repo.

It uses wget in a batch mode with the aim as being as efficient as
possible.  wget mirroring over HTTP is always going to be less efficient
than rsync, but it shouldn't be so bad since it uses --continue to check
whether it has already downloaded a file.  I suppose it could be extended
to use ETags for a little more efficiency.

I developed this creating a test mirror of f-droid.org, which is now a bit
ironic, since I added a specific check to prevent people from using this
on f-droid.org.
2017-11-29 12:40:04 +01:00

157 lines
5.8 KiB
Python
Executable File

#!/usr/bin/env python3
#
# fdroid.py - part of the FDroid server tools
# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Marti <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import logging
import fdroidserver.common
import fdroidserver.metadata
from fdroidserver import _
from argparse import ArgumentError
from collections import OrderedDict
commands = OrderedDict([
("build", _("Build a package from source")),
("init", _("Quickly start a new repository")),
("publish", _("Sign and place packages in the repo")),
("gpgsign", _("Add PGP signatures using GnuPG for packages in repo")),
("update", _("Update repo information for new packages")),
("verify", _("Verify the integrity of downloaded packages")),
("checkupdates", _("Check for updates to applications")),
("import", _("Add a new application from its source code")),
("install", _("Install built packages on devices")),
("readmeta", _("Read all the metadata files and exit")),
("rewritemeta", _("Rewrite all the metadata files")),
("lint", _("Warn about possible metadata errors")),
("scanner", _("Scan the source code of a package")),
("dscanner", _("Dynamically scan APKs post build")),
("stats", _("Update the stats of the repo")),
("server", _("Interact with the repo HTTP server")),
("signindex", _("Sign indexes created using update --nosign")),
("btlog", _("Update the binary transparency log for a URL")),
("signatures", _("Extract signatures from APKs")),
("nightly", _("Set up an app build for a nightly build repo")),
("mirror", _("Download complete mirrors of small repos")),
])
def print_help():
print(_("usage: ") + _("fdroid [-h|--help|--version] <command> [<args>]"))
print("")
print(_("Valid commands are:"))
for cmd, summary in commands.items():
print(" " + cmd + ' ' * (15 - len(cmd)) + summary)
print("")
def main():
if len(sys.argv) <= 1:
print_help()
sys.exit(0)
command = sys.argv[1]
if command not in commands:
if command in ('-h', '--help'):
print_help()
sys.exit(0)
elif command == '--version':
import os.path
output = _('no version info found!')
cmddir = os.path.realpath(os.path.dirname(__file__))
moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..')
if cmddir == moduledir:
# running from git
os.chdir(cmddir)
if os.path.isdir('.git'):
import subprocess
try:
output = subprocess.check_output(['git', 'describe'],
stderr=subprocess.STDOUT,
universal_newlines=True)
except subprocess.CalledProcessError:
output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD'],
universal_newlines=True)
elif os.path.exists('setup.py'):
import re
m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''',
open('setup.py').read(), flags=re.MULTILINE)
if m:
output = m.group(1) + '\n'
else:
from pkg_resources import get_distribution
output = get_distribution('fdroidserver').version + '\n'
print(output),
sys.exit(0)
else:
print(_("Command '%s' not recognised.\n" % command))
print_help()
sys.exit(1)
verbose = any(s in sys.argv for s in ['-v', '--verbose'])
quiet = any(s in sys.argv for s in ['-q', '--quiet'])
# Helpful to differentiate warnings from errors even when on quiet
logformat = '%(levelname)s: %(message)s'
loglevel = logging.INFO
if verbose:
loglevel = logging.DEBUG
elif quiet:
loglevel = logging.WARN
logging.basicConfig(format=logformat, level=loglevel)
if verbose and quiet:
logging.critical("Specifying --verbose and --quiet and the same time is silly")
sys.exit(1)
# Trick optparse into displaying the right usage when --help is used.
sys.argv[0] += ' ' + command
del sys.argv[1]
mod = __import__('fdroidserver.' + command, None, None, [command])
try:
mod.main()
# These are ours, contain a proper message and are "expected"
except (fdroidserver.common.FDroidException,
fdroidserver.metadata.MetaDataException) as e:
if verbose:
raise
else:
logging.critical(str(e))
sys.exit(1)
except ArgumentError as e:
logging.critical(str(e))
sys.exit(1)
except KeyboardInterrupt:
print('')
sys.exit(1)
# These should only be unexpected crashes due to bugs in the code
# str(e) often doesn't contain a reason, so just show the backtrace
except Exception as e:
logging.critical(_("Unknown exception found!"))
raise
sys.exit(0)
if __name__ == "__main__":
main()