Merge branch 'debug_flag' into 'master'

[build] Replace unused --reset-server with --debug

See merge request fdroid/fdroidserver!1038

.safety-policy.yml

@@ -17,3 +17,6 @@ security:
     62044:
       reason: "F-Droid doesn't fetch pip dependencies directly from hg/mercurial repositories: https://data.safetycli.com/v/62044/f17/"
       expires: '2025-01-31'
+    63687:
+      reason: Only affects Windows https://security-tracker.debian.org/tracker/CVE-2024-22190
+      expires: '2026-01-31'

completion/bash-completion

@@ -83,7 +83,7 @@ __complete_options() {
 __complete_build() {
 	opts="-v -q -l -s -t -f -a"
 
-	lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --scan-binary --no-tarball --force --all --no-refresh"
+	lopts="--verbose --quiet --latest --stop --test --server --debug --skip-scan --scan-binary --no-tarball --force --all --no-refresh"
 	case "${prev}" in
 		:)
 			__vercode

examples/config.yml

@@ -198,14 +198,18 @@
 # deploy_process_logs: true
 
 # The full URL to a git remote repository. You can include
-# multiple servers to mirror to by wrapping the whole thing in {} or [], and
-# including the servergitmirrors strings in a comma-separated list.
+# multiple servers to mirror to by adding strings to a YAML list or map.
 # Servers listed here will also be automatically inserted in the mirrors list.
 #
 # servergitmirrors: https://github.com/user/repo
 # servergitmirrors:
 #   - https://github.com/user/repo
 #   - https://gitlab.com/user/repo
+#
+# servergitmirrors:
+#   - url: https://github.com/user/repo
+#   - url: https://gitlab.com/user/repo
+#     indexOnly: true
 
 # Most git hosting services have hard size limits for each git repo.
 # `fdroid deploy` will delete the git history when the git mirror repo

fdroidserver/build.py

@@ -238,6 +238,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
             cmdline += ' --no-tarball'
         if (options.scan_binary or config.get('scan_binary')) and not options.skipscan:
             cmdline += ' --scan-binary'
+        if options.debug:
+            cmdline += ' --debug'
         cmdline += " %s:%s" % (app.id, build.versionCode)
         ssh_channel.exec_command('bash --login -c "' + cmdline + '"')  # nosec B601 inputs are sanitized
 
@@ -302,8 +304,12 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
     finally:
         # Suspend the build server.
         vm = vmtools.get_build_vm('builder')
-        logging.info('destroying buildserver after build')
-        vm.destroy()
+        if options.debug:
+            vm.suspend()
+            logging.warning('buildserver suspended')
+        else:
+            logging.info('destroying buildserver after build')
+            vm.destroy()
 
         # deploy logfile to repository web server
         if output:
@@ -392,15 +398,16 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                 raise BuildException("Error running sudo command for %s:%s" %
                                      (app.id, build.versionName), p.output)
 
-        p = FDroidPopen(['sudo', 'passwd', '--lock', 'root'])
-        if p.returncode != 0:
-            raise BuildException("Error locking root account for %s:%s" %
-                                 (app.id, build.versionName), p.output)
+        if not options.debug:
+            p = FDroidPopen(['sudo', 'passwd', '--lock', 'root'])
+            if p.returncode != 0:
+                raise BuildException("Error locking root account for %s:%s" %
+                                     (app.id, build.versionName), p.output)
 
-        p = FDroidPopen(['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'])
-        if p.returncode != 0:
-            raise BuildException("Error removing sudo for %s:%s" %
-                                 (app.id, build.versionName), p.output)
+            p = FDroidPopen(['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'])
+            if p.returncode != 0:
+                raise BuildException("Error removing sudo for %s:%s" %
+                                     (app.id, build.versionName), p.output)
 
         log_path = os.path.join(log_dir,
                                 common.get_toolsversion_logname(app, build))
@@ -854,8 +861,8 @@ def parse_commandline():
                         help=_("Test mode - put output in the tmp directory only, and always build, even if the output already exists."))
     parser.add_argument("--server", action="store_true", default=False,
                         help=_("Use build server"))
-    parser.add_argument("--reset-server", action="store_true", default=False,
-                        help=_("Reset and create a brand new build server, even if the existing one appears to be ok."))
+    parser.add_argument("--debug", action="store_true", default=False,
+                        help=_("Don't remove sudo and keep VM running after build (useful for debugging)."))
     # this option is internal API for telling fdroid that
     # it's running inside a buildserver vm.
     parser.add_argument("--on-server", dest="onserver", action="store_true", default=False,
@@ -929,8 +936,8 @@ def main():
 
     if config['build_server_always']:
         options.server = True
-    if options.reset_server and not options.server:
-        parser.error("option %s: Using --reset-server without --server makes no sense" % "reset-server")
+    if options.debug and not (options.server or options.onserver):
+        parser.error("option %s: Using --debug only works with --server" % "debug")
 
     log_dir = 'logs'
     if not os.path.isdir(log_dir):

fdroidserver/common.py

@@ -67,7 +67,7 @@ from pyasn1.error import PyAsn1Error
 import fdroidserver.metadata
 import fdroidserver.lint
 from fdroidserver import _
-from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
+from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException, \
     BuildException, VerificationException, MetaDataException
 from .asynchronousfilereader import AsynchronousFileReader
 from .looseversion import LooseVersion
@@ -482,8 +482,10 @@ def read_config(opts=None):
 
     if 'servergitmirrors' in config:
         if isinstance(config['servergitmirrors'], str):
-            roots = [config['servergitmirrors']]
+            roots = [{"url": config['servergitmirrors']}]
         elif all(isinstance(item, str) for item in config['servergitmirrors']):
+            roots = [{'url': i} for i in config['servergitmirrors']]
+        elif all(isinstance(item, dict) for item in config['servergitmirrors']):
             roots = config['servergitmirrors']
         else:
             raise TypeError(_('only accepts strings, lists, and tuples'))

fdroidserver/deploy.py

@@ -47,6 +47,19 @@ AUTO_S3CFG = '.fdroid-deploy-s3cfg'
 USER_S3CFG = 's3cfg'
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
+INDEX_FILES = [
+    "entry.jar",
+    "entry.json",
+    "entry.json.asc",
+    "index-v1.jar",
+    "index-v1.json",
+    "index-v1.json.asc",
+    "index-v2.json",
+    "index-v2.json.asc",
+    "index.jar",
+    "index.xml",
+]
+
 
 def _get_index_excludes(repo_section):
     """Return the list of files to be synced last, since they finalize the deploy.
@@ -447,7 +460,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
 
         enabled_remotes = []
-        for remote_url in servergitmirrors:
+        for d in servergitmirrors:
+            remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             enabled_remotes.append(name)
             r = git.remote.Remote(repo, name)
@@ -840,10 +854,9 @@ def main():
             update_serverwebroots(
                 config['serverwebroot'], repo_section, standardwebroot
             )
-        if config.get('servergitmirrors', []):
+        if config.get('servergitmirrors'):
             # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
-            servergitmirrors = config.get('servergitmirrors', [])
-            update_servergitmirrors(servergitmirrors, repo_section)
+            update_servergitmirrors(config['servergitmirrors'], repo_section)
         if config.get('awsbucket'):
             update_awsbucket(repo_section)
         if config.get('androidobservatory'):

fdroidserver/index.py

@@ -1478,7 +1478,7 @@ def add_mirrors_to_repodict(repo_section, repodict):
         repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']})
 
 
-def get_mirror_service_urls(url):
+def get_mirror_service_urls(mirror):
     """Get direct URLs from git service for use by fdroidclient.
 
     Via 'servergitmirrors', fdroidserver can create and push a mirror
@@ -1496,6 +1496,7 @@ def get_mirror_service_urls(url):
     information about the repo available to end user.
 
     """
+    url = mirror['url']
     if url.startswith('git@'):
         url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
 

fdroidserver/nightly.py

@@ -358,7 +358,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             'archive_url': repo_base + '/archive',
             'archive_description': 'Old nightly builds that have been archived.',
             'archive_older': options.archive_older,
-            'servergitmirrors': servergitmirror,
+            'servergitmirrors': [{"url": servergitmirror}],
             'keystore': KEYSTORE_FILE,
             'repo_keyalias': KEY_ALIAS,
             'keystorepass': PASSWORD,

gradlew-fdroid

@@ -187,6 +187,7 @@ get_sha() {
         '7.6.1')  echo '6147605a23b4eff6c334927a86ff3508cb5d6722cd624c97ded4c2e8640f1f87' ;;
         '7.6.2')  echo 'a01b6587e15fe7ed120a0ee299c25982a1eee045abd6a9dd5e216b2f628ef9ac' ;;
         '7.6.3')  echo '740c2e472ee4326c33bf75a5c9f5cd1e69ecf3f9b580f6e236c86d1f3d98cfac' ;;
+        '7.6.4')  echo 'bed1da33cca0f557ab13691c77f38bb67388119e4794d113e051039b80af9bb1' ;;
         '8.0')    echo '4159b938ec734a8388ce03f52aa8f3c7ed0d31f5438622545de4f83a89b79788' ;;
         '8.0.1')  echo '1b6b558be93f29438d3df94b7dfee02e794b94d9aca4611a92cdb79b6b88e909' ;;
         '8.0.2')  echo 'ff7bf6a86f09b9b2c40bb8f48b25fc19cf2b2664fd1d220cd7ab833ec758d0d7' ;;
@@ -218,7 +219,7 @@ d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6
 d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 # All gradle versions we know about
-plugin_v=(8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+plugin_v=(8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
 
 v_all=${plugin_v[@]}
 

tests/build.TestCase

@@ -470,6 +470,77 @@ class BuildTest(unittest.TestCase):
                 self.assertTrue(ndk_dir.exists())
                 self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version]))
 
+    def test_build_local_sudo(self):
+        """Test if `fdroid build --on-server` removes sudo"""
+        testdir = tempfile.mkdtemp(
+            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
+        )
+        os.chdir(testdir)
+
+        config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')}
+        fdroidserver.common.config = config
+        fdroidserver.build.config = config
+        fdroidserver.build.options = mock.Mock()
+        fdroidserver.build.options.scan_binary = False
+        fdroidserver.build.options.notarball = True
+        fdroidserver.build.options.skipscan = True
+        fdroidserver.build.options.debug = False
+
+        app = fdroidserver.metadata.App()
+        app.id = 'mocked.app.id'
+        build = fdroidserver.metadata.Build()
+        build.commit = '1.0'
+        build.output = app.id + '.apk'
+        build.versionCode = 1
+        build.versionName = '1.0'
+        vcs = mock.Mock()
+
+        def make_fake_apk(output, build):
+            with open(build.output, 'w') as fp:
+                fp.write('APK PLACEHOLDER')
+            return output
+
+        fdroidopen = mock.Mock()
+
+        fdroidopen.return_value = FakeProcess
+
+        # use "as _ignored" just to make a pretty layout
+        with mock.patch(
+            'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
+        ) as _ignored, mock.patch(
+            'fdroidserver.common.get_native_code', return_value='x86'
+        ) as _ignored, mock.patch(
+            'fdroidserver.common.get_apk_id',
+            return_value=(app.id, build.versionCode, build.versionName),
+        ) as _ignored, mock.patch(
+            'fdroidserver.common.is_apk_and_debuggable', return_value=False
+        ) as _ignored, mock.patch(
+            'fdroidserver.common.sha256sum',
+            return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
+        ) as _ignored, mock.patch(
+            'fdroidserver.common.is_apk_and_debuggable', return_value=False
+        ) as _ignored, mock.patch(
+            'fdroidserver.build.FDroidPopen', fdroidopen
+        ) as fdroidopen:
+            _ignored  # silence the linters
+            fdroidserver.build.build_local(
+                app,
+                build,
+                vcs,
+                build_dir=testdir,
+                output_dir=testdir,
+                log_dir=os.getcwd(),
+                srclib_dir=None,
+                extlib_dir=None,
+                tmp_dir=None,
+                force=False,
+                onserver=True,
+                refresh=False,
+            )
+            fdroidopen.assert_called_with(
+                ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo']
+            )
+
     def test_build_local_clean(self):
         """Test if `fdroid build` cleans ant and gradle build products"""
         os.chdir(self.testdir)

tests/deploy.TestCase

@@ -300,6 +300,86 @@ class DeployTest(unittest.TestCase):
                 name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             )
 
+    def test_update_servergitmirrors(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        update_servergitmirrors_call_iteration = 0
+        remote_push_call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        def update_servergitmirrors_call(cmd):
+            nonlocal update_servergitmirrors_call_iteration
+            if update_servergitmirrors_call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            update_servergitmirrors_call_iteration += 1
+            return 0
+
+        def remote_push_call(ref, force=False, set_upstream=False, **_args):
+            nonlocal remote_push_call_iteration
+            if remote_push_call_iteration == 0:
+                self.assertEqual([ref, force, set_upstream], ['master', True, True])
+            else:
+                self.fail('unexpected git.Remote.push invocation')
+            remote_push_call_iteration += 1
+            return []
+
+        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
+            with mock.patch(
+                'git.Remote.push', side_effect=remote_push_call
+            ) as mock_remote_push:
+                mock_remote_push.return_value = []
+                fdroidserver.deploy.update_servergitmirrors(
+                    [{'url': 'https://github.com/user/repo'}], repo_section
+                )
+        self.assertEqual(
+            update_servergitmirrors_call_iteration,
+            1,
+            'expected 1 invocations of subprocess.call',
+        )
+        self.assertEqual(
+            remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
+        )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/index.TestCase

@@ -575,7 +575,7 @@ class IndexTest(unittest.TestCase):
         ]:
             self.assertEqual(
                 ['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
-                index.get_mirror_service_urls(url),
+                index.get_mirror_service_urls({"url": url}),
             )
 
     @patch.dict(os.environ, clear=True)
@@ -603,13 +603,13 @@ class IndexTest(unittest.TestCase):
                     ]
                     self.assertEqual(
                         expected,
-                        index.get_mirror_service_urls(url),
+                        index.get_mirror_service_urls({"url": url}),
                     )
                     with patch.dict(os.environ, clear=True):
                         os.environ['CI_JOB_ID'] = ci_job_id
                         self.assertEqual(
                             expected + [artifacts_url],
-                            index.get_mirror_service_urls(url),
+                            index.get_mirror_service_urls({"url": url}),
                         )
                 with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10):
                     expected = [
@@ -617,13 +617,13 @@ class IndexTest(unittest.TestCase):
                     ]
                     self.assertEqual(
                         expected,
-                        index.get_mirror_service_urls(url),
+                        index.get_mirror_service_urls({"url": url}),
                     )
                     with patch.dict(os.environ, clear=True):
                         os.environ['CI_JOB_ID'] = ci_job_id
                         self.assertEqual(
                             expected + [artifacts_url],
-                            index.get_mirror_service_urls(url),
+                            index.get_mirror_service_urls({"url": url}),
                         )
 
     def test_make_website(self):

tests/nightly.TestCase

@@ -256,7 +256,7 @@ class NightlyTest(unittest.TestCase):
         self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
         self.assertFalse(os.path.exists('config.py'))
         git_url = 'git@github.com:f-droid/test-nightly'
-        mirror_url = index.get_mirror_service_urls(git_url)[0]
+        mirror_url = index.get_mirror_service_urls({"url": git_url})[0]
         expected = {
             'archive_description': 'Old nightly builds that have been archived.',
             'archive_name': 'f-droid/test-nightly archive',
@@ -271,7 +271,7 @@ class NightlyTest(unittest.TestCase):
             'repo_keyalias': 'androiddebugkey',
             'repo_name': 'f-droid/test-nightly',
             'repo_url': mirror_url + '/repo',
-            'servergitmirrors': git_url,
+            'servergitmirrors': [{"url": git_url}],
             'update_stats': True,
         }
         with open('config.yml') as fp:
@@ -344,7 +344,7 @@ class NightlyTest(unittest.TestCase):
             'repo_keyalias': 'androiddebugkey',
             'repo_name': 'fdroid/test-nightly',
             'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
-            'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly',
+            'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
             'update_stats': True,
         }
         with open('config.yml') as fp:

tests/run-tests

@@ -1154,7 +1154,7 @@ GIT_MIRROR=$REPOROOT/git-mirror
 cd $REPOROOT
 fdroid_init_with_prebuilt_keystore
 printf '\narchive_older: 3\n' >> config.yml
-echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml
+printf "servergitmirrors: $SERVER_GIT_MIRROR\n" >> config.yml
 
 cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/
 $fdroid update --create-metadata