Merge branch 'yml-srclib' into 'master'

yaml srclib support

Closes #609

See merge request fdroid/fdroidserver!700

CHANGELOG.md

@@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
   ([!656](https://gitlab.com/fdroid/fdroidserver/merge_requests/656))
 * add SHA256 to filename of repo graphics
   ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669))
+* support for srclibs metadata in YAML format
+  ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700))
 
 ### Fixed
 * fix build-logs dissapearing when deploying

fdroidserver/build.py

@@ -203,8 +203,18 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
             send_dir(lib)
             # Copy the metadata file too...
             ftp.chdir(posixpath.join(homedir, 'srclibs'))
-            ftp.put(os.path.join('srclibs', name + '.txt'),
+            if os.path.isfile(os.path.join('srclibs', name + '.yml')):
-                    name + '.txt')
+                ftp.put(os.path.join('srclibs', name + '.yml'),
+                        name + '.yml')
+            elif os.path.isfile(os.path.join('srclibs', name + '.txt')):
+                ftp.put(os.path.join('srclibs', name + '.txt'),
+                        name + '.txt')
+            else:
+                raise BuildException("can not find metadata file for "
+                                     "'{name}', please make sure it is "
+                                     "present in your 'srclibs' folder."
+                                     "(supported formats: txt, yml)"
+                                     .format(name=name))
         # Copy the main app source code
         # (no need if it's a srclib)
         if (not basesrclib) and os.path.exists(build_dir):

fdroidserver/common.py

@@ -773,7 +773,7 @@ def getvcs(vcstype, remote, local):
 def getsrclibvcs(name):
     if name not in fdroidserver.metadata.srclibs:
         raise VCSException("Missing srclib " + name)
-    return fdroidserver.metadata.srclibs[name]['Repo Type']
+    return fdroidserver.metadata.srclibs[name]['RepoType']
 
 
 class vcs:
@@ -1838,7 +1838,7 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
     sdir = os.path.join(srclib_dir, name)
 
     if not preponly:
-        vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
+        vcs = getvcs(srclib["RepoType"], srclib["Repo"], sdir)
         vcs.srclib = (name, number, sdir)
         if ref:
             vcs.gotorevision(ref, refresh)

fdroidserver/metadata.py

@@ -42,12 +42,15 @@ srclibs = None
 warnings_action = None
 
 
-def warn_or_exception(value):
+def warn_or_exception(value, cause=None):
     '''output warning or Exception depending on -W'''
     if warnings_action == 'ignore':
         pass
     elif warnings_action == 'error':
-        raise MetaDataException(value)
+        if cause:
+            raise MetaDataException(value) from cause
+        else:
+            raise MetaDataException(value)
     else:
         logging.warning(value)
 
@@ -704,12 +707,12 @@ def description_html(s, linkres):
     return ps.text_html
 
 
-def parse_srclib(metadatapath):
+def parse_txt_srclib(metadatapath):
 
     thisinfo = {}
 
     # Defaults for fields that come from metadata
-    thisinfo['Repo Type'] = ''
+    thisinfo['RepoType'] = ''
     thisinfo['Repo'] = ''
     thisinfo['Subdir'] = None
     thisinfo['Prepare'] = None
@@ -731,6 +734,9 @@ def parse_srclib(metadatapath):
         except ValueError:
             warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
 
+        # collapse whitespaces in field names
+        f = f.replace(' ', '')
+
         if f == "Subdir":
             thisinfo[f] = v.split(',')
         else:
@@ -741,12 +747,57 @@ def parse_srclib(metadatapath):
     return thisinfo
 
 
+def parse_yml_srclib(metadatapath):
+
+    thisinfo = {'RepoType': '',
+                'Repo': '',
+                'Subdir': None,
+                'Prepare': None}
+
+    if not os.path.exists(metadatapath):
+        warn_or_exception(_("Invalid scrlib metadata: '{file}' "
+                            "does not exist"
+                            .format(file=metadatapath)))
+        return thisinfo
+
+    with open(metadatapath, "r", encoding="utf-8") as f:
+        try:
+            data = yaml.load(f, Loader=SafeLoader)
+        except yaml.error.YAMLError as e:
+            warn_or_exception(_("Invalid srclib metadata: could not "
+                                "parse '{file}'"
+                              .format(file=metadatapath)),
+                              e)
+            return thisinfo
+
+    for key in data.keys():
+        if key not in thisinfo.keys():
+            warn_or_exception(_("Invalid srclib metadata: unknown key "
+                                "'{key}' in '{file}'")
+                              .format(key=key, file=metadatapath))
+            return thisinfo
+        else:
+            if key == 'Subdir':
+                if isinstance(data[key], str):
+                    thisinfo[key] = data[key].split(',')
+                elif isinstance(data[key], list):
+                    thisinfo[key] = data[key]
+                elif data[key] is None:
+                    thisinfo[key] = ['']
+            elif key == 'Prepare' and isinstance(data[key], list):
+                thisinfo[key] = ' && '.join(data[key])
+            else:
+                thisinfo[key] = str(data[key] or '')
+
+    return thisinfo
+
+
 def read_srclibs():
     """Read all srclib metadata.
 
     The information read will be accessible as metadata.srclibs, which is a
     dictionary, keyed on srclib name, with the values each being a dictionary
-    in the same format as that returned by the parse_srclib function.
+    in the same format as that returned by the parse_txt_srclib function.
 
     A MetaDataException is raised if there are any problems with the srclib
     metadata.
@@ -765,7 +816,11 @@ def read_srclibs():
 
     for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
         srclibname = os.path.basename(metadatapath[:-4])
-        srclibs[srclibname] = parse_srclib(metadatapath)
+        srclibs[srclibname] = parse_txt_srclib(metadatapath)
+
+    for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
+        srclibname = os.path.basename(metadatapath[:-4])
+        srclibs[srclibname] = parse_yml_srclib(metadatapath)
 
 
 def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):

tests/common.TestCase

@@ -34,7 +34,7 @@ import fdroidserver.signindex
 import fdroidserver.common
 import fdroidserver.metadata
 from testcommon import TmpCwd
-from fdroidserver.exception import FDroidException
+from fdroidserver.exception import FDroidException, VCSException
 
 
 class CommonTest(unittest.TestCase):
@@ -1241,6 +1241,63 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(123, fdroidserver.common.version_code_string_to_int('0000123'))
         self.assertEqual(-42, fdroidserver.common.version_code_string_to_int('-42'))
 
+    def test_getsrclibvcs(self):
+        fdroidserver.metadata.srclibs = {'somelib': {'RepoType': 'git'},
+                                         'yeslib': {'RepoType': 'hg'},
+                                         'nolib': {'RepoType': 'git-svn'}}
+        self.assertEqual(fdroidserver.common.getsrclibvcs('somelib'), 'git')
+        self.assertEqual(fdroidserver.common.getsrclibvcs('yeslib'), 'hg')
+        self.assertEqual(fdroidserver.common.getsrclibvcs('nolib'), 'git-svn')
+        with self.assertRaises(VCSException):
+            fdroidserver.common.getsrclibvcs('nonexistentlib')
+
+    def test_getsrclib_not_found(self):
+        fdroidserver.common.config = {'sdk_path': '',
+                                      'java_paths': {}}
+        fdroidserver.metadata.srclibs = {}
+
+        with self.assertRaisesRegex(VCSException, 'srclib SDL not found.'):
+            fdroidserver.common.getsrclib('SDL@release-2.0.3', 'srclib')
+
+    def test_getsrclib_gotorevision_raw(self):
+        fdroidserver.common.config = {'sdk_path': '',
+                                      'java_paths': {}}
+        fdroidserver.metadata.srclibs = {'SDL': {'RepoType': 'git',
+                                                 'Repo': ''}}
+
+        vcs = mock.Mock()
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.makedirs(os.path.join('srclib', 'SDL'))
+            with mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+                ret = fdroidserver.common.getsrclib('SDL', 'srclib', raw=True)
+                self.assertEqual(vcs.srclib, ('SDL', None, 'srclib/SDL'))
+                self.assertEqual(ret, vcs)
+
+    def test_getsrclib_gotorevision_ref(self):
+        fdroidserver.common.config = {'sdk_path': '',
+                                      'java_paths': {}}
+        fdroidserver.metadata.srclibs = {'ACRA': {'RepoType': 'git',
+                                                  'Repo': 'https://github.com/ACRA/acra.git',
+                                                  'Subdir': None,
+                                                  'Prepare': None}}
+
+        vcs = mock.Mock()
+        skm = mock.Mock()
+        dfm = mock.Mock()
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.makedirs(os.path.join('srclib', 'ACRA'))
+            with mock.patch('fdroidserver.common.getvcs', return_value=vcs):
+                with mock.patch('fdroidserver.common.remove_signing_keys', skm):
+                    with mock.patch('fdroidserver.common.remove_debuggable_flags', dfm):
+                        ret = fdroidserver.common.getsrclib('ACRA@acra-4.6.2', 'srclib')
+                        self.assertEqual(vcs.srclib, ('ACRA', None, 'srclib/ACRA'))
+                        vcs.gotorevision.assert_called_once_with('acra-4.6.2', True)
+                        skm.assert_called_once_with('srclib/ACRA')
+                        dfm.assert_called_once_with('srclib/ACRA')
+                        self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA'))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/metadata.TestCase

@@ -23,6 +23,8 @@ print('localmodule: ' + localmodule)
 if localmodule not in sys.path:
     sys.path.insert(0, localmodule)
 
+from testcommon import TmpCwd
+
 import fdroidserver.common
 import fdroidserver.metadata
 from fdroidserver.exception import MetaDataException
@@ -568,6 +570,274 @@ class MetadataTest(unittest.TestCase):
             UpdateCheckMode: None
             """))
 
+    def test_parse_txt_srclib(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('JSoup.txt', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # Source details (the only mandatory fields)
+                    Repo Type:git
+                    Repo:https://github.com/jhy/jsoup.git
+
+                    # Comma-separated list of subdirs to use. The first existing subdirectory
+                    # found between those given will be used. If none is found or provided, the
+                    # root of the repo directory will be used instead.
+                    Subdir:
+
+                    # Any extra commands to prepare the source library
+                    Prepare:
+                    '''))
+            srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt')
+            self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git',
+                                  'RepoType': 'git',
+                                  'Subdir': [''],
+                                  'Prepare': ''},
+                                 srclib)
+
+    def test_parse_txt_srclib_simple(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('simple.txt', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+                    Repo Type:git
+                    Repo:https://git.host/repo.git
+                    '''))
+            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
+            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
+                                  'RepoType': 'git',
+                                  'Subdir': None,
+                                  'Prepare': None},
+                                 srclib)
+
+    def test_parse_txt_srclib_simple_blanks(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('simple.txt', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+
+                    Repo Type:git
+
+                    Repo:https://git.host/repo.git
+
+                    Subdir:
+
+                    Prepare:
+                    '''))
+            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
+            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
+                                  'RepoType': 'git',
+                                  'Subdir': [''],
+                                  'Prepare': ''},
+                                 srclib)
+
+    def test_parse_txt_srclib_Changelog_cketti(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('Changelog-cketti.txt', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    Repo Type:git
+                    Repo:https://github.com/cketti/ckChangeLog
+
+                    Subdir:library,ckChangeLog/src/main
+                    Prepare:[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties
+                    '''))
+            srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt')
+            self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog',
+                                  'RepoType': 'git',
+                                  'Subdir': ['library', 'ckChangeLog/src/main'],
+                                  'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
+                                             "ant.properties && echo -e "
+                                             "'android.library=true\\ntarget=android-19' > project.properties"},
+                                 srclib)
+
+    def test_parse_yml_srclib_unknown_key(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('test.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    RepoType: git
+                    Repo: https://example.com/test.git
+                    Evil: I should not be here.
+                    '''))
+            with self.assertRaisesRegex(MetaDataException,
+                                        "Invalid srclib metadata: "
+                                        "unknown key 'Evil' in "
+                                        "'test.yml'"):
+                fdroidserver.metadata.parse_yml_srclib('test.yml')
+
+    def test_parse_yml_srclib_does_not_exists(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with self.assertRaisesRegex(MetaDataException,
+                                    "Invalid scrlib metadata: "
+                                    "'non/existent-test-srclib.yml' "
+                                    "does not exist"):
+            fdroidserver.metadata.parse_yml_srclib('non/existent-test-srclib.yml')
+
+    def test_parse_yml_srclib_simple(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('simple.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+                    RepoType: git
+                    Repo: https://git.host/repo.git
+                    '''))
+            srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml')
+            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
+                                  'RepoType': 'git',
+                                  'Subdir': None,
+                                  'Prepare': None},
+                                 srclib)
+
+    def test_parse_yml_srclib_simple_with_blanks(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('simple.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+
+                    RepoType: git
+
+                    Repo: https://git.host/repo.git
+
+                    Subdir:
+
+                    Prepare:
+                    '''))
+            srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml')
+            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
+                                  'RepoType': 'git',
+                                  'Subdir': [''],
+                                  'Prepare': ''},
+                                 srclib)
+
+    def test_parse_yml_srclib_Changelog_cketti(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('Changelog-cketti.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    RepoType: git
+                    Repo: https://github.com/cketti/ckChangeLog
+
+                    Subdir: library,ckChangeLog/src/main
+                    Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties"
+                    '''))
+            srclib = fdroidserver.metadata.parse_yml_srclib('Changelog-cketti.yml')
+            self.assertDictEqual(srclib,
+                                 {'Repo': 'https://github.com/cketti/ckChangeLog',
+                                  'RepoType': 'git',
+                                  'Subdir': ['library', 'ckChangeLog/src/main'],
+                                  'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
+                                             "ant.properties && echo -e "
+                                             "'android.library=true\\ntarget=android-19' > project.properties"})
+
+    def test_read_srclibs_yml_subdir_list(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        fdroidserver.metadata.srclibs = None
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('srclibs')
+            with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+                    RepoType: git
+                    Repo: https://git.host/repo.git
+
+                    Subdir:
+                     - This is your last chance.
+                     - After this, there is no turning back.
+                     - You take the blue pill—the story ends,
+                     - you wake up in your bed
+                     - and believe whatever you want to believe.
+                     - You take the red pill—you stay in Wonderland
+                     - and I show you how deep the rabbit-hole goes.
+                    Prepare:
+                        There is a difference between knowing the path
+                        and walking the path.
+                    '''))
+            fdroidserver.metadata.read_srclibs()
+        self.maxDiff = None
+        self.assertDictEqual(fdroidserver.metadata.srclibs,
+                             {'with-list': {'RepoType': 'git',
+                                            'Repo': 'https://git.host/repo.git',
+                                            'Subdir': ['This is your last chance.',
+                                                       'After this, there is no turning back.',
+                                                       'You take the blue pill—the story ends,',
+                                                       'you wake up in your bed',
+                                                       'and believe whatever you want to believe.',
+                                                       'You take the red pill—you stay in Wonderland',
+                                                       'and I show you how deep the rabbit-hole goes.'],
+                                            'Prepare': 'There is a difference between knowing the path '
+                                                       'and walking the path.'}})
+
+    def test_read_srclibs_yml_prepare_list(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        fdroidserver.metadata.srclibs = None
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('srclibs')
+            with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+                    RepoType: git
+                    Repo: https://git.host/repo.git
+
+                    Subdir:
+                    Prepare:
+                     - The Matrix is a system, Neo.
+                     - That system is our enemy.
+                     - But when you're inside, you look around, what do you see?
+                     - Businessmen, teachers, lawyers, carpenters.
+                     - The very minds of the people we are trying to save.
+                     - But until we do, these people are still a part of that system and that makes them our enemy.
+                     - You have to understand, most of these people are not ready to be unplugged.
+                     - And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.
+                    '''))
+            fdroidserver.metadata.read_srclibs()
+        self.maxDiff = None
+        self.assertDictEqual(fdroidserver.metadata.srclibs,
+                             {'with-list': {'RepoType': 'git',
+                                            'Repo': 'https://git.host/repo.git',
+                                            'Subdir': [''],
+                                            'Prepare': 'The Matrix is a system, Neo. && '
+                                                       'That system is our enemy. && '
+                                                       'But when you\'re inside, you look around, what do you see? && '
+                                                       'Businessmen, teachers, lawyers, carpenters. && '
+                                                       'The very minds of the people we are trying to save. && '
+                                                       'But until we do, these people are still a part of that system and that makes them our enemy. && '
+                                                       'You have to understand, most of these people are not ready to be unplugged. && '
+                                                       'And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.'}})
+
+    def test_read_srclibs(self):
+        fdroidserver.metadata.warnings_action = 'error'
+        fdroidserver.metadata.srclibs = None
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('srclibs')
+            with open('srclibs/simple.yml', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    RepoType: git
+                    Repo: https://git.host/repo.git
+                    '''))
+            with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f:
+                f.write(textwrap.dedent('''\
+                    # this should be simple
+                    Repo Type:git
+                    Repo:https://git.host/repo.git
+
+                    Subdir:
+                    Prepare:
+                    '''))
+            fdroidserver.metadata.read_srclibs()
+        self.assertDictEqual(fdroidserver.metadata.srclibs,
+                             {'simple-wb': {'RepoType': 'git',
+                                            'Repo': 'https://git.host/repo.git',
+                                            'Subdir': [''],
+                                            'Prepare': ''},
+                              'simple': {'RepoType': 'git',
+                                         'Repo': 'https://git.host/repo.git',
+                                         'Subdir': None,
+                                         'Prepare': None}})
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))