establish config/mirrors.yml as a way to set up mirrors

The mirrors: entry in config.yml is great for quick access and shorter
mirror lists.  Now that we are adding a lot more metadata to the mirrors,
including potentially the full text of the privacy policy, having this
in its own file should make these cases easier to manage.

examples/config.yml

@@ -228,6 +228,9 @@
 #     countryCode: BA
 #   - url: http://foobarfoobarfoobar.onion/fdroid
 #
+# The list of mirrors can also be maintained in config/mirrors.yml, a
+# standalone YAML file in the optional configuration directory.  In
+# that case, mirrors: should be removed from this file (config.yml).
 
 
 # optionally specify which identity file to use when using rsync or git over SSH

fdroidserver/common.py

@@ -117,11 +117,13 @@ GITLAB_COM_PAGES_MAX_SIZE = 1000000000
 ANTIFEATURES_CONFIG_NAME = 'antiFeatures'
 CATEGORIES_CONFIG_NAME = 'categories'
 CONFIG_CONFIG_NAME = 'config'
+MIRRORS_CONFIG_NAME = 'mirrors'
 RELEASECHANNELS_CONFIG_NAME = "releaseChannels"
 CONFIG_NAMES = (
     ANTIFEATURES_CONFIG_NAME,
     CATEGORIES_CONFIG_NAME,
     CONFIG_CONFIG_NAME,
+    MIRRORS_CONFIG_NAME,
     RELEASECHANNELS_CONFIG_NAME,
 )
 

fdroidserver/index.py

@@ -29,6 +29,7 @@ import re
 import shutil
 import tempfile
 import urllib.parse
+import yaml
 import zipfile
 import calendar
 import qrcode
@@ -42,7 +43,7 @@ from . import common
 from . import metadata
 from . import net
 from . import signindex
-from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
+from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
 from fdroidserver.exception import FDroidException, VerificationException
 
 
@@ -1399,6 +1400,17 @@ def add_mirrors_to_repodict(repo_section, repodict):
     if type(mirrors_config) not in (list, tuple):
         mirrors_config = [mirrors_config]
 
+    mirrors_yml = Path(f'config/{MIRRORS_CONFIG_NAME}.yml')
+    if mirrors_yml.exists():
+        if mirrors_config:
+            raise FDroidException(
+                _('mirrors set twice, in config.yml and {path}!').format(
+                    path=mirrors_yml
+                )
+            )
+        with mirrors_yml.open() as fp:
+            mirrors_config = yaml.safe_load(fp)
+
     mirrorcheckfailed = False
     mirrors = []
     urls = set()

tests/index.TestCase

@@ -791,6 +791,43 @@ class IndexTest(unittest.TestCase):
             with self.assertRaises(fdroidserver.exception.FDroidException):
                 index.add_mirrors_to_repodict('repo', repodict)
 
+    def test_load_mirrors_config_from_file(self):
+        # empty the dict for *.config, see setUp()
+        for k in sorted(common.config.keys()):
+            del common.config[k]
+
+        os.chdir(self.testdir)
+        os.mkdir('config')
+        primary = 'https://primary.com/fdroid/repo'
+        mirror = 'https://mirror.com/fdroid'
+        with open('config/mirrors.yml', 'w') as fp:
+            yaml.dump([{'url': mirror}], fp)
+        repodict = {'address': primary}
+        index.add_mirrors_to_repodict('repo', repodict)
+        self.assertEqual(
+            repodict['mirrors'],
+            [
+                {'isPrimary': True, 'url': primary},
+                {'url': mirror + '/repo'},
+            ],
+        )
+
+    def test_error_when_load_mirrors_from_config_and_file(self):
+        # empty the dict for *.config, see setUp()
+        for k in sorted(common.config.keys()):
+            del common.config[k]
+
+        os.chdir(self.testdir)
+        os.mkdir('config')
+        with open('config/mirrors.yml', 'w') as fp:
+            yaml.dump([{'url': 'https://foo.com'}], fp)
+        repodict = {
+            'address': 'https://foo.com',
+            'mirrors': {'url': 'http://two/fdroid/repo'},
+        }
+        with self.assertRaises(fdroidserver.exception.FDroidException):
+            index.add_mirrors_to_repodict('repo', repodict)
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))