git_mirror_size_limit config option to set max git mirror size

GitHub and GitLab have some kinds of limits on how big a git repo can be,
this makes that option configurable.  This also is very useful for tests.

examples/config.py

@@ -190,6 +190,13 @@ The repository of older versions of applications from the main demo repository.
 #     'https://gitlab.com/user/repo',
 #     }
 
+# Most git hosting services have hard size limits for each git repo.
+# `fdroid deploy` will delete the git history when the git mirror repo
+# approaches this limit to ensure that the repo will still fit when
+# pushed.  GitHub recommends 1GB, gitlab.com recommends 10GB.
+#
+# git_mirror_size_limit = '10GB'
+
 # Any mirrors of this repo, for example all of the servers declared in
 # serverwebroot and all the servers declared in servergitmirrors,
 # will automatically be used by the client.  If one

fdroidserver/common.py

@@ -148,6 +148,7 @@ default_config = {
     'archive_description': _('These are the apps that have been archived from the main repo.'),
     'archive_older': 0,
     'lint_licenses': fdroidserver.lint.APPROVED_LICENSES,
+    'git_mirror_size_limit': 10000000000,
 }
 
 
@@ -354,9 +355,31 @@ def read_config(opts, config_file='config.py'):
             raise TypeError(_('only accepts strings, lists, and tuples'))
         config['servergitmirrors'] = roots
 
+        limit = config['git_mirror_size_limit']
+        config['git_mirror_size_limit'] = parse_human_readable_size(limit)
+
     return config
 
 
+def parse_human_readable_size(size):
+    units = {
+        'b': 1,
+        'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4,
+        'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4,
+    }
+    try:
+        return int(float(size))
+    except (ValueError, TypeError):
+        if type(size) != str:
+            raise ValueError(_('Could not parse size "{size}", wrong type "{type}"')
+                             .format(size=size, type=type(size)))
+        s = size.lower().replace(' ', '')
+        m = re.match(r'^(?P<value>[0-9][0-9.]+) *(?P<unit>' + r'|'.join(units.keys()) + r')$', s)
+        if not m:
+            raise ValueError(_('Not a valid size definition: "{}"').format(size))
+        return int(float(m.group("value")) * units[m.group("unit")])
+
+
 def assert_config_keystore(config):
     """Check weather keystore is configured correctly and raise exception if not."""
 

fdroidserver/server.py

@@ -356,11 +356,15 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         # github/gitlab use bare git repos, so only count the .git folder
         # test: generate giant APKs by including AndroidManifest.xml and and large
         # file from /dev/urandom, then sign it.  Then add those to the git repo.
-        if os.path.isdir(dotgit) and _get_size(dotgit) > 1000000000:
+        dotgit_size = _get_size(dotgit)
-            logging.warning('Deleting git-mirror history, repo is too big (1 gig max)')
+        dotgit_over_limit = dotgit_size > config['git_mirror_size_limit']
+        if os.path.isdir(dotgit) and dotgit_over_limit:
+            logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})')
+                            .format(size=dotgit_size, limit=config['git_mirror_size_limit']))
             shutil.rmtree(dotgit)
-        if options.no_keep_git_mirror_archive and _get_size(dotgit) > 1000000000:
+        if options.no_keep_git_mirror_archive and dotgit_over_limit:
-            logging.warning('Deleting archive, repo is too big (1 gig max)')
+            logging.warning(_('Deleting archive, repo is too big ({size} max {limit})')
+                            .format(size=dotgit_size, limit=config['git_mirror_size_limit']))
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 

tests/common.TestCase

@@ -45,6 +45,15 @@ class CommonTest(unittest.TestCase):
             os.makedirs(self.tmpdir)
         os.chdir(self.basedir)
 
+    def test_parse_human_readable_size(self):
+        for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1),
+                     ('10.43 KiB', 10680), ('11GB', 11000000000), ('59kb', 59000),
+                     ('343.1 mb', 343100000), ('99.9GiB', 107266808217)):
+            self.assertEqual(fdroidserver.common.parse_human_readable_size(k), v)
+        for v in ((12, 123), '0xfff', [], None, '12,123', '123GG', '982374bb', self):
+            with self.assertRaises(ValueError):
+                fdroidserver.common.parse_human_readable_size(v)
+
     def test_assert_config_keystore(self):
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with self.assertRaises(FDroidException):

tests/run-tests

@@ -1096,6 +1096,59 @@ $fdroid update --create-key
 test -e $KEYSTORE
 
 
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror"
+
+# fake git remote server for repo mirror
+SERVER_GIT_MIRROR=`create_test_dir`
+cd $SERVER_GIT_MIRROR
+git init
+git config receive.denyCurrentBranch updateInstead
+
+REPOROOT=`create_test_dir`
+GIT_MIRROR=$REPOROOT/git-mirror
+cd $REPOROOT
+fdroid_init_with_prebuilt_keystore
+echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py
+
+cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/
+$fdroid update --create-metadata
+$fdroid deploy
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_3.apk
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_3.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
+date > $GIT_MIRROR/.git/test-stamp
+
+# add one more APK to trigger archiving
+cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo/
+$fdroid update
+$fdroid deploy
+test -e $REPOROOT/archive/com.politedroid_3.apk
+! test -e $GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
+! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
+test -e $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
+test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk
+before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'`
+
+echo "git_mirror_size_limit = '60kb'" >> config.py
+$fdroid update
+$fdroid deploy
+test -e $REPOROOT/archive/com.politedroid_3.apk
+! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
+after=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'`
+! test -e $GIT_MIRROR/.git/test-stamp
+git -C $GIT_MIRROR gc
+git -C $SERVER_GIT_MIRROR gc
+test $before -gt $after
+
+
 #------------------------------------------------------------------------------#
 echo_header "sign binary repo in offline box, then publishing from online box"