fdroidserver/tests/deploy.TestCase

#!/usr/bin/env python3

import configparser
import inspect
import logging
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock

localmodule = os.path.realpath(
    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
if localmodule not in sys.path:
    sys.path.insert(0, localmodule)

import fdroidserver.common
import fdroidserver.deploy
from fdroidserver.exception import FDroidException
from testcommon import TmpCwd, mkdtemp, parse_args_for_test


class Options:
    quiet = False
    verbose = False


class DeployTest(unittest.TestCase):
    '''fdroidserver/deploy.py'''

    def setUp(self):
        logging.basicConfig(level=logging.DEBUG)
        self.basedir = os.path.join(localmodule, 'tests')
        os.chdir(self.basedir)
        self._td = mkdtemp()
        self.testdir = self._td.name

        fdroidserver.deploy.options = mock.Mock()
        fdroidserver.deploy.config = {}
        fdroidserver.deploy.USER_RCLONE_CONF = False

    def tearDown(self):
        self._td.cleanup()

    def test_update_serverwebroots_bad_None(self):
        with self.assertRaises(TypeError):
            fdroidserver.deploy.update_serverwebroots(None, 'repo')

    def test_update_serverwebroots_bad_int(self):
        with self.assertRaises(TypeError):
            fdroidserver.deploy.update_serverwebroots(9, 'repo')

    def test_update_serverwebroots_bad_float(self):
        with self.assertRaises(TypeError):
            fdroidserver.deploy.update_serverwebroots(1.0, 'repo')

    def test_update_serverwebroots(self):
        """rsync works with file paths, so this test uses paths for the URLs"""
        os.chdir(self.testdir)
        repo = Path('repo')
        repo.mkdir()
        fake_apk = repo / 'fake.apk'
        with fake_apk.open('w') as fp:
            fp.write('not an APK, but has the right filename')
        url0 = Path('url0/fdroid')
        url0.mkdir(parents=True)
        url1 = Path('url1/fdroid')
        url1.mkdir(parents=True)

        dest_apk0 = url0 / fake_apk
        dest_apk1 = url1 / fake_apk
        self.assertFalse(dest_apk0.is_file())
        self.assertFalse(dest_apk1.is_file())
        fdroidserver.deploy.update_serverwebroots(
            [
                {'url': str(url0)},
                {'url': str(url1)},
            ],
            str(repo),
        )
        self.assertTrue(dest_apk0.is_file())
        self.assertTrue(dest_apk1.is_file())

    def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
        with self.assertRaises(SystemExit):
            fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')

    def test_update_serverwebroots_bad_ssh_url(self):
        with self.assertRaises(SystemExit):
            fdroidserver.deploy.update_serverwebroots(
                [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
            )

    def test_update_serverwebroots_unsupported_ssh_url(self):
        with self.assertRaises(SystemExit):
            fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')

    @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
    def test_update_remote_storage_with_rclone(self):
        os.chdir(self.testdir)
        repo = Path('repo')
        repo.mkdir(parents=True, exist_ok=True)

        fake_apk = repo / 'another_fake.apk'
        with fake_apk.open('w') as fp:
            fp.write('not an APK, but has the right filename')

        # write out rclone config for test use
        rclone_config = configparser.ConfigParser()
        rclone_config.add_section("test-local-config")
        rclone_config.set("test-local-config", "type", "local")

        rclone_config_path = Path('rclone_config_path')
        rclone_config_path.mkdir(parents=True, exist_ok=True)
        rclone_file = rclone_config_path / 'rclone.conf'
        with open(rclone_file, 'w') as configfile:
            rclone_config.write(configfile)

        # setup parameters for this test run
        fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
        fdroidserver.deploy.config['rclone'] = True
        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
        fdroidserver.deploy.options = Options

        # write out destination path
        destination = Path('some_bucket_folder/fdroid')
        destination.mkdir(parents=True, exist_ok=True)
        dest_path = Path(destination) / fake_apk
        self.assertFalse(dest_path.is_file())
        repo_section = str(repo)
        # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
        fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
        self.assertFalse(dest_path.is_file())

    def test_update_serverwebroot(self):
        """rsync works with file paths, so this test uses paths for the URLs"""
        os.chdir(self.testdir)
        repo = Path('repo')
        repo.mkdir(parents=True)
        fake_apk = repo / 'fake.apk'
        with fake_apk.open('w') as fp:
            fp.write('not an APK, but has the right filename')
        url = Path('url')
        url.mkdir()

        dest_apk = url / fake_apk
        self.assertFalse(dest_apk.is_file())
        fdroidserver.deploy.options = mock.Mock()
        fdroidserver.deploy.options.identity_file = None
        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
        self.assertTrue(dest_apk.is_file())

    @mock.patch.dict(os.environ, clear=True)
    def test_update_serverwebroot_no_rsync_error(self):
        os.environ['PATH'] = self.testdir
        os.chdir(self.testdir)
        with self.assertRaises(FDroidException):
            fdroidserver.deploy.update_serverwebroot('serverwebroot', 'repo')

    def test_update_serverwebroot_make_cur_version_link(self):
        self.maxDiff = None

        # setup parameters for this test run
        fdroidserver.common.options = mock.Mock()
        fdroidserver.common.options.no_checksum = True
        fdroidserver.common.options.identity_file = None
        fdroidserver.common.options.verbose = False
        fdroidserver.common.options.quiet = True
        fdroidserver.deploy.config = {'make_current_version_link': True}
        url = "example.com:/var/www/fdroid"
        repo_section = 'repo'

        # setup function for asserting subprocess.call invocations
        call_iteration = 0

        def update_server_webroot_call(cmd):
            nonlocal call_iteration
            if call_iteration == 0:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--archive',
                        '--delete-after',
                        '--safe-links',
                        '--quiet',
                        '--exclude',
                        'repo/altstore-index.json',
                        '--exclude',
                        'repo/entry.jar',
                        '--exclude',
                        'repo/entry.json',
                        '--exclude',
                        'repo/entry.json.asc',
                        '--exclude',
                        'repo/index-v1.jar',
                        '--exclude',
                        'repo/index-v1.json',
                        '--exclude',
                        'repo/index-v1.json.asc',
                        '--exclude',
                        'repo/index-v2.json',
                        '--exclude',
                        'repo/index-v2.json.asc',
                        '--exclude',
                        'repo/index.jar',
                        '--exclude',
                        'repo/index.xml',
                        'repo',
                        'example.com:/var/www/fdroid',
                    ],
                )
            elif call_iteration == 1:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--archive',
                        '--delete-after',
                        '--safe-links',
                        '--quiet',
                        'repo',
                        url,
                    ],
                )
            elif call_iteration == 2:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--archive',
                        '--delete-after',
                        '--safe-links',
                        '--quiet',
                        'Sym.apk',
                        'Sym.apk.asc',
                        'Sym.apk.sig',
                        'example.com:/var/www/fdroid',
                    ],
                )
            else:
                self.fail('unexpected subprocess.call invocation')
            call_iteration += 1
            return 0

        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('repo')
            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
            with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
        self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')

    def test_update_serverwebroot_with_id_file(self):
        # setup parameters for this test run
        fdroidserver.common.options = mock.Mock()
        fdroidserver.common.options.identity_file = None
        fdroidserver.common.options.no_checksum = True
        fdroidserver.common.options.verbose = True
        fdroidserver.common.options.quiet = False
        fdroidserver.common.options.identity_file = None
        fdroidserver.deploy.config = {'identity_file': './id_rsa'}
        url = "example.com:/var/www/fdroid"
        repo_section = 'archive'

        # setup function for asserting subprocess.call invocations
        call_iteration = 0

        def update_server_webroot_call(cmd):
            nonlocal call_iteration
            if call_iteration == 0:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--archive',
                        '--delete-after',
                        '--safe-links',
                        '--verbose',
                        '-e',
                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                        + fdroidserver.deploy.config['identity_file'],
                        '--exclude',
                        'archive/altstore-index.json',
                        '--exclude',
                        'archive/entry.jar',
                        '--exclude',
                        'archive/entry.json',
                        '--exclude',
                        'archive/entry.json.asc',
                        '--exclude',
                        'archive/index-v1.jar',
                        '--exclude',
                        'archive/index-v1.json',
                        '--exclude',
                        'archive/index-v1.json.asc',
                        '--exclude',
                        'archive/index-v2.json',
                        '--exclude',
                        'archive/index-v2.json.asc',
                        '--exclude',
                        'archive/index.jar',
                        '--exclude',
                        'archive/index.xml',
                        'archive',
                        url,
                    ],
                )
            elif call_iteration == 1:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--archive',
                        '--delete-after',
                        '--safe-links',
                        '--verbose',
                        '-e',
                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                        + fdroidserver.deploy.config['identity_file'],
                        'archive',
                        url,
                    ],
                )
            else:
                self.fail('unexpected subprocess.call invocation')
            call_iteration += 1
            return 0

        with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
        self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')

    @unittest.skipIf(
        not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
    )
    def test_upload_to_virustotal(self):
        fdroidserver.common.options.verbose = True
        virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
        fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)

    def test_remote_hostname_regex(self):
        for remote_url, name in (
            ('git@github.com:guardianproject/fdroid-repo', 'github'),
            ('git@gitlab.com:guardianproject/fdroid-repo', 'gitlab'),
            ('https://github.com:guardianproject/fdroid-repo', 'github'),
            ('https://gitlab.com/guardianproject/fdroid-repo', 'gitlab'),
            ('https://salsa.debian.org/foo/repo', 'salsa'),
        ):
            self.assertEqual(
                name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
            )

    def test_update_servergitmirrors(self):
        # setup parameters for this test run
        fdroidserver.common.options = mock.Mock()
        fdroidserver.common.options.identity_file = None
        fdroidserver.common.options.no_keep_git_mirror_archive = False
        fdroidserver.common.options.verbose = False
        fdroidserver.common.options.quiet = True
        fdroidserver.common.options.index_only = False

        config = {}
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.deploy.config = config
        fdroidserver.deploy.config["servergitmirrors"] = []

        repo_section = 'repo'

        # setup function for asserting subprocess.call invocations
        update_servergitmirrors_call_iteration = 0
        remote_push_call_iteration = 0

        os.chdir(self.testdir)
        repo = Path('repo')
        repo.mkdir(parents=True)
        fake_apk = repo / 'Sym.apk'
        with fake_apk.open('w') as fp:
            fp.write('not an APK, but has the right filename')
        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
        with fake_index.open('w') as fp:
            fp.write('not an index, but has the right filename')

        def update_servergitmirrors_call(cmd):
            nonlocal update_servergitmirrors_call_iteration
            if update_servergitmirrors_call_iteration == 0:
                self.assertListEqual(
                    cmd,
                    [
                        'rsync',
                        '--recursive',
                        '--safe-links',
                        '--times',
                        '--perms',
                        '--one-file-system',
                        '--delete',
                        '--chmod=Da+rx,Fa-x,a+r,u+w',
                        '--quiet',
                        'repo/',
                        "git-mirror/fdroid/repo/",
                    ],
                )
            else:
                self.fail('unexpected subprocess.call invocation')
            update_servergitmirrors_call_iteration += 1
            return 0

        def remote_push_call(ref, force=False, set_upstream=False, **_args):
            nonlocal remote_push_call_iteration
            if remote_push_call_iteration == 0:
                self.assertEqual([ref, force, set_upstream], ['master', True, True])
            else:
                self.fail('unexpected git.Remote.push invocation')
            remote_push_call_iteration += 1
            return []

        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
            with mock.patch(
                'git.Remote.push', side_effect=remote_push_call
            ) as mock_remote_push:
                mock_remote_push.return_value = []
                fdroidserver.deploy.update_servergitmirrors(
                    [{'url': 'https://github.com/user/repo'}], repo_section
                )
        self.assertEqual(
            update_servergitmirrors_call_iteration,
            1,
            'expected 1 invocations of subprocess.call',
        )
        self.assertEqual(
            remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
        )


if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))

    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        default=False,
        help="Spew out even more information than normal",
    )
    parse_args_for_test(parser, sys.argv)

    newSuite = unittest.TestSuite()
    newSuite.addTest(unittest.makeSuite(DeployTest))
    unittest.main(failfast=False)