test(deploy): add test cases for update_awsbucket_s3cmd

fdroidserver/deploy.py

@@ -190,13 +190,13 @@ def update_awsbucket_s3cmd(repo_section):
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
         sync_indexes_flags.append('--delete-removed')
         sync_indexes_flags.append('--delete-after')
         if options.no_checksum:
             sync_indexes_flags.append('--no-check-md5')
         else:
             sync_indexes_flags.append('--check-md5')
-        sync_indexes_flags.extend(_get_index_includes(repo_section))
         returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
         if returncode != 0:
             raise FDroidException()

tests/deploy.TestCase

@@ -70,6 +70,11 @@ class DeployTest(unittest.TestCase):
         url1 = Path('url1/fdroid')
         url1.mkdir(parents=True)
 
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
         dest_apk0 = url0 / fake_apk
         dest_apk1 = url1 / fake_apk
         self.assertFalse(dest_apk0.is_file())
@@ -176,20 +181,20 @@ class DeployTest(unittest.TestCase):
         fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
-        serverwebroot = Path('serverwebroot')
+        url = Path('url')
-        serverwebroot.mkdir()
+        url.mkdir()
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = False
 
-        dest_apk = Path(serverwebroot) / fake_apk
+        dest_apk = Path(url) / fake_apk
-        dest_index = Path(serverwebroot) / fake_index
+        dest_index = Path(url) / fake_index
         self.assertFalse(dest_apk.is_file())
         self.assertFalse(dest_index.is_file())
 
-        fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
+        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertFalse(dest_apk.is_file())
         self.assertTrue(dest_index.is_file())
 
@@ -298,6 +303,7 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.verbose = False
@@ -305,7 +311,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = True
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
         # setup function for asserting subprocess.call invocations
@@ -360,7 +366,7 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         'repo',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 2:
@@ -389,7 +395,7 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_serverwebroot_with_id_file(self):
@@ -482,7 +488,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
         # setup function for asserting subprocess.call invocations
@@ -527,7 +533,7 @@ class DeployTest(unittest.TestCase):
                         '--exclude',
                         "*",
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 1:
@@ -543,7 +549,7 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             else:
@@ -552,7 +558,7 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_localcopy_in_index_only_mode(self):
@@ -568,6 +574,7 @@ class DeployTest(unittest.TestCase):
         call_iteration = 0
 
         with tempfile.TemporaryDirectory() as local_copy_dir:
+
             def update_localcopy_call(cmd):
                 nonlocal call_iteration
                 if call_iteration == 0:
@@ -575,8 +582,13 @@ class DeployTest(unittest.TestCase):
                         cmd,
                         [
                             'rsync',
-                            '--recursive', '--safe-links', '--times', '--perms',
+                            '--recursive',
-                            '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w',
+                            '--safe-links',
+                            '--times',
+                            '--perms',
+                            '--one-file-system',
+                            '--delete',
+                            '--chmod=Da+rx,Fa-x,a+r,u+w',
                             '--verbose',
                             '--include',
                             "*/",
@@ -715,6 +727,221 @@ class DeployTest(unittest.TestCase):
             remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
         )
 
+    def test_update_awsbucket_s3cmd(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        '--skip-existing',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 3:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
+
+    def test_update_awsbucket_s3cmd_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--include',
+                        'repo/entry.jar',
+                        '--include',
+                        'repo/entry.json',
+                        '--include',
+                        'repo/entry.json.asc',
+                        '--include',
+                        'repo/index-v1.jar',
+                        '--include',
+                        'repo/index-v1.json',
+                        '--include',
+                        'repo/index-v1.json.asc',
+                        '--include',
+                        'repo/index-v2.json',
+                        '--include',
+                        'repo/index-v2.json.asc',
+                        '--include',
+                        'repo/index.jar',
+                        '--include',
+                        'repo/index.xml',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))