From e76a0c9d6afc35474b1b7773978e7b46278210f2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 13 Jan 2020 11:48:23 +0100 Subject: [PATCH] git_mirror_size_limit config option to set max git mirror size GitHub and GitLab have some kinds of limits on how big a git repo can be, this makes that option configurable. This also is very useful for tests. --- examples/config.py | 7 ++++++ fdroidserver/common.py | 23 ++++++++++++++++++ fdroidserver/server.py | 12 ++++++---- tests/common.TestCase | 9 +++++++ tests/run-tests | 53 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/examples/config.py b/examples/config.py index b400af6c..9035baa5 100644 --- a/examples/config.py +++ b/examples/config.py @@ -190,6 +190,13 @@ The repository of older versions of applications from the main demo repository. # 'https://gitlab.com/user/repo', # } +# Most git hosting services have hard size limits for each git repo. +# `fdroid deploy` will delete the git history when the git mirror repo +# approaches this limit to ensure that the repo will still fit when +# pushed. GitHub recommends 1GB, gitlab.com recommends 10GB. +# +# git_mirror_size_limit = '10GB' + # Any mirrors of this repo, for example all of the servers declared in # serverwebroot and all the servers declared in servergitmirrors, # will automatically be used by the client. If one diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 05060658..a18d49f1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -148,6 +148,7 @@ default_config = { 'archive_description': _('These are the apps that have been archived from the main repo.'), 'archive_older': 0, 'lint_licenses': fdroidserver.lint.APPROVED_LICENSES, + 'git_mirror_size_limit': 10000000000, } @@ -354,9 +355,31 @@ def read_config(opts, config_file='config.py'): raise TypeError(_('only accepts strings, lists, and tuples')) config['servergitmirrors'] = roots + limit = config['git_mirror_size_limit'] + config['git_mirror_size_limit'] = parse_human_readable_size(limit) + return config +def parse_human_readable_size(size): + units = { + 'b': 1, + 'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4, + 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4, + } + try: + return int(float(size)) + except (ValueError, TypeError): + if type(size) != str: + raise ValueError(_('Could not parse size "{size}", wrong type "{type}"') + .format(size=size, type=type(size))) + s = size.lower().replace(' ', '') + m = re.match(r'^(?P[0-9][0-9.]+) *(?P' + r'|'.join(units.keys()) + r')$', s) + if not m: + raise ValueError(_('Not a valid size definition: "{}"').format(size)) + return int(float(m.group("value")) * units[m.group("unit")]) + + def assert_config_keystore(config): """Check weather keystore is configured correctly and raise exception if not.""" diff --git a/fdroidserver/server.py b/fdroidserver/server.py index a69ed030..18b30f7c 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -356,11 +356,15 @@ def update_servergitmirrors(servergitmirrors, repo_section): # github/gitlab use bare git repos, so only count the .git folder # test: generate giant APKs by including AndroidManifest.xml and and large # file from /dev/urandom, then sign it. Then add those to the git repo. - if os.path.isdir(dotgit) and _get_size(dotgit) > 1000000000: - logging.warning('Deleting git-mirror history, repo is too big (1 gig max)') + dotgit_size = _get_size(dotgit) + dotgit_over_limit = dotgit_size > config['git_mirror_size_limit'] + if os.path.isdir(dotgit) and dotgit_over_limit: + logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) shutil.rmtree(dotgit) - if options.no_keep_git_mirror_archive and _get_size(dotgit) > 1000000000: - logging.warning('Deleting archive, repo is too big (1 gig max)') + if options.no_keep_git_mirror_archive and dotgit_over_limit: + logging.warning(_('Deleting archive, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) diff --git a/tests/common.TestCase b/tests/common.TestCase index 33db7283..3d90707a 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -45,6 +45,15 @@ class CommonTest(unittest.TestCase): os.makedirs(self.tmpdir) os.chdir(self.basedir) + def test_parse_human_readable_size(self): + for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1), + ('10.43 KiB', 10680), ('11GB', 11000000000), ('59kb', 59000), + ('343.1 mb', 343100000), ('99.9GiB', 107266808217)): + self.assertEqual(fdroidserver.common.parse_human_readable_size(k), v) + for v in ((12, 123), '0xfff', [], None, '12,123', '123GG', '982374bb', self): + with self.assertRaises(ValueError): + fdroidserver.common.parse_human_readable_size(v) + def test_assert_config_keystore(self): with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with self.assertRaises(FDroidException): diff --git a/tests/run-tests b/tests/run-tests index 7ae22fe9..d769686e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -1096,6 +1096,59 @@ $fdroid update --create-key test -e $KEYSTORE +#------------------------------------------------------------------------------# +echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror" + +# fake git remote server for repo mirror +SERVER_GIT_MIRROR=`create_test_dir` +cd $SERVER_GIT_MIRROR +git init +git config receive.denyCurrentBranch updateInstead + +REPOROOT=`create_test_dir` +GIT_MIRROR=$REPOROOT/git-mirror +cd $REPOROOT +fdroid_init_with_prebuilt_keystore +echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py + +cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ +$fdroid update --create-metadata +$fdroid deploy +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_3.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +date > $GIT_MIRROR/.git/test-stamp + +# add one more APK to trigger archiving +cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo/ +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk +test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk +before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` + +echo "git_mirror_size_limit = '60kb'" >> config.py +$fdroid update +$fdroid deploy +test -e $REPOROOT/archive/com.politedroid_3.apk +! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk +after=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` +! test -e $GIT_MIRROR/.git/test-stamp +git -C $GIT_MIRROR gc +git -C $SERVER_GIT_MIRROR gc +test $before -gt $after + + #------------------------------------------------------------------------------# echo_header "sign binary repo in offline box, then publishing from online box"