fix all bandit B310 urllib_urlopen

"Audit url open for permitted schemes. Allowing use of ‘file:’’ or custom
schemes is often unexpected."

https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b310-urllib-urlopen

.gitlab-ci.yml

@@ -140,7 +140,7 @@ lint_format_safety_bandit_checks:
     - ./hooks/pre-commit || export EXITVALUE=1
     - bandit
         -ii
-        -s B110,B310,B322,B404,B408,B410,B603,B607
+        -s B110,B322,B404,B408,B410,B603,B607
         -r $CI_PROJECT_DIR fdroid
         || export EXITVALUE=1
     - safety check --full-report || export EXITVALUE=1

fdroidserver/checkupdates.py

@@ -64,7 +64,7 @@ def check_http(app):
         if len(urlcode) > 0:
             logging.debug("...requesting {0}".format(urlcode))
             req = urllib.request.Request(urlcode, None)
-            resp = urllib.request.urlopen(req, None, 20)
+            resp = urllib.request.urlopen(req, None, 20)  # nosec B310 scheme is filtered above
             page = resp.read().decode('utf-8')
 
             m = re.search(codeex, page)
@@ -77,7 +77,7 @@ def check_http(app):
             if urlver != '.':
                 logging.debug("...requesting {0}".format(urlver))
                 req = urllib.request.Request(urlver, None)
-                resp = urllib.request.urlopen(req, None, 20)
+                resp = urllib.request.urlopen(req, None, 20)  # nosec B310 scheme is filtered above
                 page = resp.read().decode('utf-8')
 
             m = re.search(verex, page)
@@ -295,7 +295,7 @@ def check_gplay(app):
     headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
     req = urllib.request.Request(url, None, headers)
     try:
-        resp = urllib.request.urlopen(req, None, 20)
+        resp = urllib.request.urlopen(req, None, 20)  # nosec B310 URL base is hardcoded above
         page = resp.read().decode()
     except urllib.error.HTTPError as e:
         return (None, str(e.code))

fdroidserver/import.py

@@ -40,8 +40,9 @@ SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
 # when one of these is found it's assumed that's the information we want.
 # Returns repotype, address, or None, reason
 def getrepofrompage(url):
-
-    req = urllib.request.urlopen(url)
+    if not url.startswith('http'):
+        return (None, _('{url} does not start with "http"!'.format(url=url)))
+    req = urllib.request.urlopen(url)  # nosec B310 non-http URLs are filtered out
     if req.getcode() != 200:
         return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
     page = req.read().decode(req.headers.get_content_charset())

tests/checkupdates.TestCase

@@ -19,6 +19,7 @@ if localmodule not in sys.path:
 
 import fdroidserver.checkupdates
 import fdroidserver.metadata
+from fdroidserver.exception import FDroidException
 
 
 class CommonTest(unittest.TestCase):
@@ -123,6 +124,17 @@ class CommonTest(unittest.TestCase):
             self.assertEqual(vername, '1.1.9')
             self.assertEqual(vercode, '10109')
 
+    def test_check_http_blocks_unknown_schemes(self):
+        app = fdroidserver.metadata.App()
+        for scheme in ('file', 'ssh', 'http', ';pwn'):
+            app.id = scheme
+            faked = scheme + '://fake.url/for/testing/scheme'
+            app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored'
+            app.metadatapath = 'metadata/' + app.id + '.yml'
+            vername, vercode = fdroidserver.checkupdates.check_http(app)
+            self.assertIsNone(vername)
+            self.assertTrue(FDroidException.__name__ in vercode)
+
     def test_check_http_ignore(self):
         fdroidserver.checkupdates.options = mock.Mock()