Merge branch 'checkupdates-merge-request' into 'master'

checkupdates: --merge-request flag for single app runs then a MR

See merge request fdroid/fdroidserver!1149

fdroidserver/checkupdates.py

@@ -18,6 +18,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import git
 import os
 import re
 import urllib.request
@@ -683,6 +684,110 @@ def get_last_build_from_app(app: metadata.App) -> metadata.Build:
         return metadata.Build()
 
 
+def push_commits(remote_name='origin', verbose=False):
+    """Make git branch then push commits as merge request.
+
+    This uses the appid as the standard branch name so that there is
+    only ever one open merge request per-app.  If multiple apps are
+    included in the branch, then 'checkupdates' is used as branch
+    name.  This is to support the old way operating, e.g. in batches.
+
+    This uses GitLab "Push Options" to create a merge request. Git
+    Push Options are config data that can be sent via `git push
+    --push-option=... origin foo`.
+
+    References
+    ----------
+    * https://docs.gitlab.com/ee/user/project/push_options.html
+
+    """
+    git_repo = git.Repo.init('.')
+    files = set()
+    upstream_main = 'main' if 'main' in git_repo.remotes.upstream.refs else 'master'
+    local_main = 'main' if 'main' in git_repo.refs else 'master'
+    for commit in git_repo.iter_commits(f'upstream/{upstream_main}...{local_main}'):
+        files.update(commit.stats.files.keys())
+
+    branch_name = 'checkupdates'
+    files = list(files)
+    if len(files) == 1:
+        m = re.match(r'metadata/([^\s]+)\.yml', files[0])
+        if m:
+            branch_name = m.group(1)  # appid
+    if not files:
+        return
+    progress = None
+    if verbose:
+        import clint.textui
+
+        progress_bar = clint.textui.progress.Bar()
+
+        class MyProgressPrinter(git.RemoteProgress):
+            def update(self, op_code, current, maximum=None, message=None):
+                if isinstance(maximum, float):
+                    progress_bar.show(current, maximum)
+
+        progress = MyProgressPrinter()
+
+    git_repo.create_head(branch_name, force=True)
+    remote = git_repo.remotes[remote_name]
+    pushinfos = remote.push(
+        branch_name, force=True, set_upstream=True, progress=progress
+    )
+    pushinfos = remote.push(
+        branch_name,
+        progress=progress,
+        force=True,
+        set_upstream=True,
+        push_option=[
+            'merge_request.create',
+            'merge_request.remove_source_branch',
+            'merge_request.title=' + 'bot: checkupdates for ' + branch_name,
+            'merge_request.description='
+            + 'checkupdates-bot run %s' % os.getenv('CI_JOB_URL'),
+        ],
+    )
+
+    for pushinfo in pushinfos:
+        if pushinfo.flags & (
+            git.remote.PushInfo.ERROR
+            | git.remote.PushInfo.REJECTED
+            | git.remote.PushInfo.REMOTE_FAILURE
+            | git.remote.PushInfo.REMOTE_REJECTED
+        ):
+            # Show potentially useful messages from git remote
+            if progress:
+                for line in progress.other_lines:
+                    if line.startswith('remote:'):
+                        logging.debug(line)
+            raise FDroidException(
+                f'{remote.url} push failed: {pushinfo.flags} {pushinfo.summary}'
+            )
+        else:
+            logging.debug(remote.url + ': ' + pushinfo.summary)
+
+
+def prune_empty_appid_branches(git_repo=None):
+    """Remove empty branches from checkupdates-bot git remote."""
+    if git_repo is None:
+        git_repo = git.Repo.init('.')
+    main_branch = 'main'
+    if main_branch not in git_repo.remotes.upstream.refs:
+        main_branch = 'master'
+    upstream_main = 'upstream/' + main_branch
+
+    remote = git_repo.remotes.origin
+    remote.update(prune=True)
+    merged_branches = git_repo.git().branch(remotes=True, merged=upstream_main).split()
+    for remote_branch in merged_branches:
+        if not remote_branch or '/' not in remote_branch:
+            continue
+        if remote_branch.split('/')[1] not in (main_branch, 'HEAD'):
+            for ref in git_repo.remotes.origin.refs:
+                if remote_branch == ref.name:
+                    remote.push(':%s' % ref.remote_head, force=True)  # rm remote branch
+
+
 def status_update_json(processed: list, failed: dict) -> None:
     """Output a JSON file with metadata about this run."""
     logging.debug(_('Outputting JSON'))
@@ -716,6 +821,8 @@ def main():
                         help=_("Only process apps with auto-updates"))
     parser.add_argument("--commit", action="store_true", default=False,
                         help=_("Commit changes"))
+    parser.add_argument("--merge-request", action="store_true", default=False,
+                        help=_("Commit changes, push, then make a merge request"))
     parser.add_argument("--allow-dirty", action="store_true", default=False,
                         help=_("Run on git repo that has uncommitted changes"))
     metadata.add_metadata_arguments(parser)
@@ -730,6 +837,10 @@ def main():
             logging.error(_('Build metadata git repo has uncommited changes!'))
             sys.exit(1)
 
+    if options.merge_request and not (options.appid and len(options.appid) == 1):
+        logging.error(_('--merge-request only runs on a single appid!'))
+        sys.exit(1)
+
     apps = common.read_app_args(options.appid)
 
     processed = []
@@ -745,7 +856,7 @@ def main():
         logging.info(msg)
 
         try:
-            checkupdates_app(app, options.auto, options.commit)
+            checkupdates_app(app, options.auto, options.commit or options.merge_request)
             processed.append(appid)
         except Exception as e:
             msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
@@ -754,6 +865,10 @@ def main():
             failed[appid] = str(e)
             exit_code = 1
 
+    if options.appid and options.merge_request:
+        push_commits(verbose=options.verbose)
+        prune_empty_appid_branches()
+
     status_update_json(processed, failed)
     sys.exit(exit_code)
 

fdroidserver/deploy.py

@@ -805,9 +805,10 @@ def upload_to_servergitmirror(
                 | git.remote.PushInfo.REMOTE_REJECTED
             ):
                 # Show potentially useful messages from git remote
-                for line in progress.other_lines:
+                if progress:
-                    if line.startswith('remote:'):
+                    for line in progress.other_lines:
-                        logging.debug(line)
+                        if line.startswith('remote:'):
+                            logging.debug(line)
                 raise FDroidException(
                     remote.url
                     + ' push failed: '

pyproject.toml

@@ -49,7 +49,6 @@ force-exclude = '''(
   | tests/gradle-release-checksums\.py
   | tests/openssl-version-check-test\.py
   | tests/valid-package-names/test\.py
-  | tests/checkupdates\.TestCase
   | tests/common\.TestCase
   | tests/publish\.TestCase
   | tests/signatures\.TestCase

tests/checkupdates.TestCase

@@ -2,9 +2,13 @@
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
+import git
 import logging
 import os
+import shutil
 import sys
+import tempfile
+import time
 import unittest
 from unittest import mock
 from pathlib import Path
@@ -27,6 +31,12 @@ class CheckupdatesTest(unittest.TestCase):
         logging.basicConfig(level=logging.DEBUG)
         self.basedir = localmodule / 'tests'
         os.chdir(self.basedir)
+        self.testdir = tempfile.TemporaryDirectory(
+            str(time.time()), self._testMethodName + '_'
+        )
+
+    def tearDown(self):
+        self.testdir.cleanup()
 
     def test_autoupdatemode_no_suffix(self):
         fdroidserver.checkupdates.config = {}
@@ -257,9 +267,9 @@ class CheckupdatesTest(unittest.TestCase):
         vcs.latesttags.return_value = ['1.1.9', '1.1.8']
         with mock.patch(
             'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
-        ) as _ignored, mock.patch.object(
+        ) as _ignored, mock.patch.object(Path, 'is_file') as mock_path, mock.patch(
-            Path, 'is_file'
+            'fdroidserver.common.getvcs', return_value=vcs
-        ) as mock_path, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+        ):
             _ignored  # silence the linters
             mock_path.is_file.return_falue = True
             vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
@@ -269,9 +279,9 @@ class CheckupdatesTest(unittest.TestCase):
         app.UpdateCheckData = r'b.txt|c(.*)|.|v(.*)'
         with mock.patch(
             'pathlib.Path.read_text', lambda a: 'v1.1.0\nc10109'
-        ) as _ignored, mock.patch.object(
+        ) as _ignored, mock.patch.object(Path, 'is_file') as mock_path, mock.patch(
-            Path, 'is_file'
+            'fdroidserver.common.getvcs', return_value=vcs
-        ) as mock_path, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+        ):
             _ignored  # silence the linters
             mock_path.is_file.return_falue = True
             vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
@@ -281,9 +291,9 @@ class CheckupdatesTest(unittest.TestCase):
         app.UpdateCheckData = r'b.txt|c(.*)||'
         with mock.patch(
             'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
-        ) as _ignored, mock.patch.object(
+        ) as _ignored, mock.patch.object(Path, 'is_file') as mock_path, mock.patch(
-            Path, 'is_file'
+            'fdroidserver.common.getvcs', return_value=vcs
-        ) as mock_path, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+        ):
             _ignored  # silence the linters
             mock_path.is_file.return_falue = True
             vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
@@ -294,9 +304,9 @@ class CheckupdatesTest(unittest.TestCase):
         app.UpdateCheckData = r'b.txt|c(.*)||Android-([\d.]+)'
         with mock.patch(
             'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
-        ) as _ignored, mock.patch.object(
+        ) as _ignored, mock.patch.object(Path, 'is_file') as mock_path, mock.patch(
-            Path, 'is_file'
+            'fdroidserver.common.getvcs', return_value=vcs
-        ) as mock_path, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+        ):
             _ignored  # silence the linters
             mock_path.is_file.return_falue = True
             vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
@@ -317,6 +327,142 @@ class CheckupdatesTest(unittest.TestCase):
         self.assertEqual(vername, '2')
         self.assertEqual(vercode, 2)
 
+    def _get_test_git_repos(self):
+        testdir = self.testdir.name
+        os.chdir(testdir)
+        os.mkdir('metadata')
+        for f in (self.basedir / 'metadata').glob('*.yml'):
+            shutil.copy(f, 'metadata')
+        git_repo = git.Repo.init(testdir)
+        git_repo.git.add(all=True)
+        git_repo.index.commit("all metadata files")
+
+        git_remote_upstream = os.path.join(testdir, 'git_remote_upstream')
+        upstream_repo = git.Repo.init(git_remote_upstream, bare=True)
+        with upstream_repo.config_writer() as cw:
+            cw.set_value('receive', 'advertisePushOptions', True)
+        git_repo.create_remote('upstream', 'file://' + git_remote_upstream)
+
+        git_remote_origin = os.path.join(testdir, 'git_remote_origin')
+        origin_repo = git.Repo.init(git_remote_origin, bare=True)
+        with origin_repo.config_writer() as cw:
+            cw.set_value('receive', 'advertisePushOptions', True)
+        git_repo.create_remote('origin', 'file://' + git_remote_origin)
+
+        return git_repo, origin_repo, upstream_repo
+
+    def test_push_commits(self):
+        git_repo, origin_repo, upstream_repo = self._get_test_git_repos()
+        for remote in git_repo.remotes:
+            remote.push(git_repo.active_branch)
+        self.assertEqual(git_repo.head, upstream_repo.head)
+        self.assertEqual(origin_repo.head, upstream_repo.head)
+        # pretend that checkupdates ran but didn't create any new commits
+        fdroidserver.checkupdates.push_commits()
+
+        appid = 'org.adaway'
+        self.assertNotIn(appid, git_repo.branches)
+        self.assertNotIn(appid, origin_repo.branches)
+        self.assertNotIn(appid, upstream_repo.branches)
+        self.assertNotIn('checkupdates', git_repo.branches)
+
+        # now make commit
+        app = fdroidserver.metadata.read_metadata({appid: -1})[appid]
+        build = fdroidserver.metadata.Build()
+        build.versionName = 'fake'
+        build.versionCode = 999999999
+        app.Builds.append(build)
+        metadata_file = 'metadata/%s.yml' % appid
+        fdroidserver.metadata.write_metadata(metadata_file, app)
+        git_repo.index.add(metadata_file)
+        git_repo.index.commit('changed ' + appid)
+
+        # and push the new commit to the dynamic branch
+        fdroidserver.checkupdates.push_commits()
+        self.assertIn(appid, git_repo.branches)
+        self.assertIn(appid, git_repo.remotes.origin.refs)
+        self.assertNotIn('checkupdates', git_repo.branches)
+        self.assertNotIn(appid, git_repo.remotes.upstream.refs)
+
+    def test_push_commits_verbose(self):
+        class Options:
+            verbose = True
+
+        fdroidserver.checkupdates.options = Options
+        repos = self._get_test_git_repos()
+        git_repo = repos[0]
+        git_repo.remotes.origin.push(git_repo.active_branch)
+        git_repo.remotes.upstream.push(git_repo.active_branch)
+
+        # make commit
+        appid = 'org.adaway'
+        app = fdroidserver.metadata.read_metadata({appid: -1})[appid]
+        build = fdroidserver.metadata.Build()
+        build.versionName = 'fake'
+        build.versionCode = 999999999
+        app.Builds.append(build)
+        metadata_file = 'metadata/%s.yml' % appid
+        fdroidserver.metadata.write_metadata(metadata_file, app)
+        git_repo.index.add(metadata_file)
+        git_repo.index.commit('changed ' + appid)
+
+        # and push the new commit to the dynamic branch
+        fdroidserver.checkupdates.push_commits()
+        self.assertIn(appid, git_repo.branches)
+        self.assertIn(appid, git_repo.remotes.origin.refs)
+
+    def test_prune_empty_appid_branches(self):
+        git_repo, origin_repo, upstream_repo = self._get_test_git_repos()
+        for remote in git_repo.remotes:
+            remote.push(git_repo.active_branch)
+        self.assertEqual(git_repo.head, upstream_repo.head)
+        self.assertEqual(origin_repo.head, upstream_repo.head)
+
+        appid = 'org.adaway'
+        git_repo.create_head(appid, force=True)
+        git_repo.remotes.origin.push(appid, force=True)
+        self.assertIn(appid, git_repo.branches)
+        self.assertIn(appid, origin_repo.branches)
+        self.assertIn(appid, git_repo.remotes.origin.refs)
+        self.assertNotIn(appid, git_repo.remotes.upstream.refs)
+        fdroidserver.checkupdates.prune_empty_appid_branches()
+        self.assertNotIn(appid, origin_repo.branches)
+        self.assertNotIn(appid, git_repo.remotes.origin.refs)
+        self.assertNotIn(appid, git_repo.remotes.upstream.refs)
+
+    @mock.patch('sys.exit')
+    @mock.patch('fdroidserver.metadata.read_metadata')
+    def test_merge_requests_flag(self, read_metadata, sys_exit):
+        def _sys_exit(return_code=0):
+            assert return_code != 0
+            raise fdroidserver.exception.FDroidException('sys.exit() ran')
+
+        def _read_metadata(a=None, b=None):
+            raise StopIteration('read_metadata() ran, test is successful')
+
+        appid = 'com.example'
+        # read_metadata.return_value = dict()  # {appid: dict()}
+        read_metadata.side_effect = _read_metadata
+        sys_exit.side_effect = _sys_exit
+
+        # set up clean git repo
+        os.chdir(self.testdir.name)
+        git_repo = git.Repo.init()
+        open('foo', 'w').close()
+        git_repo.git.add(all=True)
+        git_repo.index.commit("all files")
+
+        with mock.patch('sys.argv', ['fdroid checkupdates', '--merge-request']):
+            with self.assertRaises(fdroidserver.exception.FDroidException):
+                fdroidserver.checkupdates.main()
+        sys_exit.assert_called()
+
+        sys_exit.reset_mock()
+        with mock.patch('sys.argv', ['fdroid checkupdates', '--merge-request', appid]):
+            with self.assertRaises(StopIteration):
+                fdroidserver.checkupdates.main()
+        sys_exit.assert_not_called()
+
 
 if __name__ == "__main__":
     import argparse