diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a11a82e6..e0aced08 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -111,6 +111,10 @@ MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647) XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}' +# https://docs.gitlab.com/ee/user/gitlab_com/#gitlab-pages +GITLAB_COM_PAGES_MAX_SIZE = 1000000000 + + config = None options = None env = None @@ -478,6 +482,13 @@ def parse_human_readable_size(size): return int(float(m.group("value")) * units[m.group("unit")]) +def get_dir_size(path_or_str): + """Get the total size of all files in the given directory.""" + if isinstance(path_or_str, str): + path_or_str = Path(path_or_str) + return sum(f.stat().st_size for f in path_or_str.glob('**/*') if f.is_file()) + + def assert_config_keystore(config): """Check weather keystore is configured correctly and raise exception if not.""" nosigningkey = False diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 01930409..380bebb4 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -1003,11 +1003,19 @@ def get_mirror_service_urls(url): """Get direct URLs from git service for use by fdroidclient. Via 'servergitmirrors', fdroidserver can create and push a mirror - to certain well known git services like gitlab or github. This + to certain well known git services like GitLab or GitHub. This will always use the 'master' branch since that is the default branch in git. The files are then accessible via alternate URLs, where they are served in their raw format via a CDN rather than from git. + + Both of the GitLab URLs will work with F-Droid, but only the + GitLab Pages will work in the browser This is because the "raw" + URLs are not served with the correct mime types, so any index.html + which is put in the repo will not be rendered. Putting an + index.html file in the repo root is a common way for to make + information about the repo available to end user. + """ if url.startswith('git@'): url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url) @@ -1030,14 +1038,17 @@ def get_mirror_service_urls(url): segments.extend([branch, folder]) urls.append('/'.join(segments)) elif hostname == "gitlab.com": - # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser - # This is because the `raw` URLs are not served with the correct mime types, so any - # index.html which is put in the repo will not be rendered. Putting an index.html file in - # the repo root is a common way for to make information about the repo available to end user. - - # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" - gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] - urls.append('/'.join(gitlab_pages)) + if common.get_dir_size() <= common.GITLAB_COM_PAGES_MAX_SIZE: + # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" + gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] + urls.append('/'.join(gitlab_pages)) + else: + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder" gitlab_raw = segments + ['-', 'raw', branch, folder] urls.append('/'.join(gitlab_raw)) diff --git a/tests/index.TestCase b/tests/index.TestCase index e9220a23..f399f8fc 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -358,7 +358,7 @@ class IndexTest(unittest.TestCase): ) self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml'))) - def test_get_mirror_service_urls(self): + def test_github_get_mirror_service_urls(self): for url in [ 'git@github.com:foo/bar', 'git@github.com:foo/bar.git', @@ -370,19 +370,28 @@ class IndexTest(unittest.TestCase): fdroidserver.index.get_mirror_service_urls(url), ) + def test_gitlab_get_mirror_service_urls(self): for url in [ 'git@gitlab.com:group/project', 'git@gitlab.com:group/project.git', 'https://gitlab.com/group/project', 'https://gitlab.com/group/project.git', ]: - self.assertEqual( - [ - 'https://group.gitlab.io/project/fdroid', - 'https://gitlab.com/group/project/-/raw/master/fdroid', - ], - fdroidserver.index.get_mirror_service_urls(url), - ) + with patch('fdroidserver.common.get_dir_size', return_value=100000): + self.assertEqual( + [ + 'https://group.gitlab.io/project/fdroid', + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ], + fdroidserver.index.get_mirror_service_urls(url), + ) + with patch('fdroidserver.common.get_dir_size', return_value=1234567890): + self.assertEqual( + [ + 'https://gitlab.com/group/project/-/raw/master/fdroid', + ], + fdroidserver.index.get_mirror_service_urls(url), + ) def test_make_website(self): tmptestsdir = tempfile.mkdtemp(