Merge branch 'yamllint' into 'master'

yamllint

See merge request fdroid/fdroidserver!721

CHANGELOG.md

@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
   ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669))
 * support for srclibs metadata in YAML format
   ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700))
+* check srclibs and app-metadata files with yamllint
+  ([!721](https://gitlab.com/fdroid/fdroidserver/merge_requests/721))
 
 ### Fixed
 * fix build-logs dissapearing when deploying

fdroidserver/common.py

@@ -63,7 +63,7 @@ import fdroidserver.metadata
 import fdroidserver.lint
 from fdroidserver import _
 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
-    BuildException, VerificationException
+    BuildException, VerificationException, MetaDataException
 from .asynchronousfilereader import AsynchronousFileReader
 
 # The path to this fdroidserver distribution
@@ -1809,6 +1809,36 @@ def get_app_from_url(url):
     return app
 
 
+def parse_srclib_spec(spec):
+
+    if type(spec) != str:
+        raise MetaDataException(_("can not parse scrlib spec "
+                                  "(not a string): '{}'")
+                                .format(spec))
+
+    tokens = spec.split('@')
+    if len(tokens) > 2:
+        raise MetaDataException(_("could not parse srclib spec "
+                                  "(too many '@' signs): '{}'")
+                                .format(spec))
+    elif len(tokens) < 2:
+        raise MetaDataException(_("could not parse srclib spec "
+                                  "(no ref specified): '{}'")
+                                .format(spec))
+
+    name = tokens[0]
+    ref = tokens[1]
+    number = None
+    subdir = None
+
+    if ':' in name:
+        number, name = name.split(':', 1)
+    if '/' in name:
+        name, subdir = name.split('/', 1)
+
+    return (name, ref, number, subdir)
+
+
 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
               raw=False, prepare=True, preponly=False, refresh=True,
               build=None):
@@ -3735,3 +3765,25 @@ def force_exit(exitvalue=0):
     sys.stdout.flush()
     sys.stderr.flush()
     os._exit(exitvalue)
+
+
+YAML_LINT_CONFIG = {'extends': 'default',
+                    'rules': {'document-start': 'disable',
+                              'line-length': 'disable',
+                              'truthy': 'disable'}}
+
+
+def run_yamllint(path, indent=0):
+
+    try:
+        import yamllint.config
+        import yamllint.linter
+    except ImportError:
+        return ''
+
+    result = []
+    with open(path, 'r', encoding='utf-8') as f:
+        problems = yamllint.linter.run(f, yamllint.config.YamlLintConfig(json.dumps(YAML_LINT_CONFIG)))
+    for problem in problems:
+        result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message)
+    return '\n'.join(result)

fdroidserver/lint.py

@@ -574,6 +574,9 @@ def main():
     common.setup_global_opts(parser)
     parser.add_argument("-f", "--format", action="store_true", default=False,
                         help=_("Also warn about formatting issues, like rewritemeta -l"))
+    parser.add_argument('--force-yamllint', action="store_true", default=False,
+                        help=_("When linting the entire repository yamllint is disabled by default. "
+                               "This option forces yamllint regardless."))
     parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
@@ -600,6 +603,29 @@ def main():
         if app.Disabled:
             continue
 
+        # only run yamllint when linting individual apps.
+        if len(options.appid) > 0 or options.force_yamllint:
+
+            # run yamllint on app metadata
+            ymlpath = os.path.join('metadata', appid + '.yml')
+            if os.path.isfile(ymlpath):
+                yamllintresult = common.run_yamllint(ymlpath)
+                if yamllintresult != '':
+                    print(yamllintresult)
+
+            # run yamllint on srclib metadata
+            srclibs = set()
+            for build in app.builds:
+                for srclib in build.srclibs:
+                    srclibs.add(srclib)
+            for srclib in srclibs:
+                name, ref, number, subdir = common.parse_srclib_spec(srclib)
+                srclibpath = os.path.join('srclibs', name + '.yml')
+                if os.path.isfile(srclibpath):
+                    yamllintresult = common.run_yamllint(srclibpath)
+                    if yamllintresult != '':
+                        print(yamllintresult)
+
         app_check_funcs = [
             check_app_field_types,
             check_regexes,

fdroidserver/metadata.py

@@ -747,7 +747,7 @@ def parse_txt_srclib(metadatapath):
     return thisinfo
 
 
-def parse_yml_srclib(metadatapath):
+def parse_yaml_srclib(metadatapath):
 
     thisinfo = {'RepoType': '',
                 'Repo': '',
@@ -765,9 +765,11 @@ def parse_yml_srclib(metadatapath):
             data = yaml.load(f, Loader=SafeLoader)
         except yaml.error.YAMLError as e:
             warn_or_exception(_("Invalid srclib metadata: could not "
-                                "parse '{file}'"
+                                "parse '{file}'")
-                              .format(file=metadatapath)),
+                              .format(file=metadatapath) + '\n'
-                              e)
+                              + fdroidserver.common.run_yamllint(metadatapath,
+                                                                 indent=4),
+                              cause=e)
             return thisinfo
 
     for key in data.keys():
@@ -820,7 +822,7 @@ def read_srclibs():
 
     for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
         srclibname = os.path.basename(metadatapath[:-4])
-        srclibs[srclibname] = parse_yml_srclib(metadatapath)
+        srclibs[srclibname] = parse_yaml_srclib(metadatapath)
 
 
 def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
@@ -1102,7 +1104,14 @@ def parse_json_metadata(mf, app):
 
 
 def parse_yaml_metadata(mf, app):
-    yamldata = yaml.load(mf, Loader=SafeLoader)
+    try:
+        yamldata = yaml.load(mf, Loader=SafeLoader)
+    except yaml.YAMLError as e:
+        warn_or_exception(_("could not parse '{path}'")
+                          .format(path=mf.name) + '\n'
+                          + fdroidserver.common.run_yamllint(mf.name,
+                                                             indent=4),
+                          cause=e)
 
     deprecated_in_yaml = ['Provides']
 

setup.py

@@ -86,6 +86,7 @@ setup(name='fdroidserver',
           'qrcode',
           'ruamel.yaml >= 0.15',
           'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
+          'yamllint',
       ],
       classifiers=[
           'Development Status :: 4 - Beta',

tests/common.TestCase

@@ -34,7 +34,7 @@ import fdroidserver.signindex
 import fdroidserver.common
 import fdroidserver.metadata
 from testcommon import TmpCwd
-from fdroidserver.exception import FDroidException, VCSException
+from fdroidserver.exception import FDroidException, VCSException, MetaDataException
 
 
 class CommonTest(unittest.TestCase):
@@ -1014,6 +1014,22 @@ class CommonTest(unittest.TestCase):
             subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
             self.assertEqual(subdirs[f], subdir)
 
+    def test_parse_srclib_spec_good(self):
+        self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'),
+                         ('osmand-external-skia', 'android/oreo', None, None))
+        self.assertEqual(fdroidserver.common.parse_srclib_spec('1:appcompat@v7'),
+                         ('appcompat', 'v7', '1', None))
+        self.assertEqual(fdroidserver.common.parse_srclib_spec('1:Support/v7/appcompat@android-4.4_r1.1'),
+                         ('Support', 'android-4.4_r1.1', '1', 'v7/appcompat'))
+
+    def test_parse_srclib_spec_bad(self):
+        with self.assertRaises(MetaDataException):
+            self.assertEqual(fdroidserver.common.parse_srclib_spec(None))
+        with self.assertRaises(MetaDataException):
+            self.assertEqual(fdroidserver.common.parse_srclib_spec('no-ref'))
+        with self.assertRaises(MetaDataException):
+            self.assertEqual(fdroidserver.common.parse_srclib_spec('@multi@at-signs@'))
+
     def test_bad_urls(self):
         for url in ('asdf',
                     'file://thing.git',
@@ -1298,6 +1314,32 @@ class CommonTest(unittest.TestCase):
                         dfm.assert_called_once_with('srclib/ACRA')
                         self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA'))
 
+    def test_run_yamllint_wellformed(self):
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('wellformed.yml', 'w') as f:
+                f.write(textwrap.dedent('''\
+                    yaml:
+                        file:
+                            - for
+                            - test
+                        purposeses: true
+                    '''))
+            result = fdroidserver.common.run_yamllint('wellformed.yml')
+            self.assertEqual(result, '')
+
+    def test_run_yamllint_malformed(self):
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            with open('malformed.yml', 'w') as f:
+                f.write(textwrap.dedent('''\
+                    yaml:
+                          - that
+                     fails
+                          - test
+                    '''))
+            result = fdroidserver.common.run_yamllint('malformed.yml')
+            self.assertIsNotNone(result)
+            self.assertNotEqual(result, '')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/metadata.TestCase

@@ -652,7 +652,7 @@ class MetadataTest(unittest.TestCase):
                                              "'android.library=true\\ntarget=android-19' > project.properties"},
                                  srclib)
 
-    def test_parse_yml_srclib_unknown_key(self):
+    def test_parse_yaml_srclib_unknown_key(self):
         fdroidserver.metadata.warnings_action = 'error'
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('test.yml', 'w', encoding='utf-8') as f:
@@ -665,17 +665,17 @@ class MetadataTest(unittest.TestCase):
                                         "Invalid srclib metadata: "
                                         "unknown key 'Evil' in "
                                         "'test.yml'"):
-                fdroidserver.metadata.parse_yml_srclib('test.yml')
+                fdroidserver.metadata.parse_yaml_srclib('test.yml')
 
-    def test_parse_yml_srclib_does_not_exists(self):
+    def test_parse_yaml_srclib_does_not_exists(self):
         fdroidserver.metadata.warnings_action = 'error'
         with self.assertRaisesRegex(MetaDataException,
                                     "Invalid scrlib metadata: "
                                     "'non/existent-test-srclib.yml' "
                                     "does not exist"):
-            fdroidserver.metadata.parse_yml_srclib('non/existent-test-srclib.yml')
+            fdroidserver.metadata.parse_yaml_srclib('non/existent-test-srclib.yml')
 
-    def test_parse_yml_srclib_simple(self):
+    def test_parse_yaml_srclib_simple(self):
         fdroidserver.metadata.warnings_action = 'error'
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('simple.yml', 'w', encoding='utf-8') as f:
@@ -684,14 +684,14 @@ class MetadataTest(unittest.TestCase):
                     RepoType: git
                     Repo: https://git.host/repo.git
                     '''))
-            srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml')
+            srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml')
             self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                   'RepoType': 'git',
                                   'Subdir': None,
                                   'Prepare': None},
                                  srclib)
 
-    def test_parse_yml_srclib_simple_with_blanks(self):
+    def test_parse_yaml_srclib_simple_with_blanks(self):
         fdroidserver.metadata.warnings_action = 'error'
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('simple.yml', 'w', encoding='utf-8') as f:
@@ -706,14 +706,14 @@ class MetadataTest(unittest.TestCase):
 
                     Prepare:
                     '''))
-            srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml')
+            srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml')
             self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                   'RepoType': 'git',
                                   'Subdir': [''],
                                   'Prepare': ''},
                                  srclib)
 
-    def test_parse_yml_srclib_Changelog_cketti(self):
+    def test_parse_yaml_srclib_Changelog_cketti(self):
         fdroidserver.metadata.warnings_action = 'error'
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('Changelog-cketti.yml', 'w', encoding='utf-8') as f:
@@ -724,7 +724,7 @@ class MetadataTest(unittest.TestCase):
                     Subdir: library,ckChangeLog/src/main
                     Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties"
                     '''))
-            srclib = fdroidserver.metadata.parse_yml_srclib('Changelog-cketti.yml')
+            srclib = fdroidserver.metadata.parse_yaml_srclib('Changelog-cketti.yml')
             self.assertDictEqual(srclib,
                                  {'Repo': 'https://github.com/cketti/ckChangeLog',
                                   'RepoType': 'git',