diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ff73dc..de4c33a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669)) * support for srclibs metadata in YAML format ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700)) +* check srclibs and app-metadata files with yamllint + ([!721](https://gitlab.com/fdroid/fdroidserver/merge_requests/721)) ### Fixed * fix build-logs dissapearing when deploying diff --git a/fdroidserver/common.py b/fdroidserver/common.py index f0699f58..4ce6d86a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -63,7 +63,7 @@ import fdroidserver.metadata import fdroidserver.lint from fdroidserver import _ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\ - BuildException, VerificationException + BuildException, VerificationException, MetaDataException from .asynchronousfilereader import AsynchronousFileReader # The path to this fdroidserver distribution @@ -1809,6 +1809,36 @@ def get_app_from_url(url): return app +def parse_srclib_spec(spec): + + if type(spec) != str: + raise MetaDataException(_("can not parse scrlib spec " + "(not a string): '{}'") + .format(spec)) + + tokens = spec.split('@') + if len(tokens) > 2: + raise MetaDataException(_("could not parse srclib spec " + "(too many '@' signs): '{}'") + .format(spec)) + elif len(tokens) < 2: + raise MetaDataException(_("could not parse srclib spec " + "(no ref specified): '{}'") + .format(spec)) + + name = tokens[0] + ref = tokens[1] + number = None + subdir = None + + if ':' in name: + number, name = name.split(':', 1) + if '/' in name: + name, subdir = name.split('/', 1) + + return (name, ref, number, subdir) + + def getsrclib(spec, srclib_dir, subdir=None, basepath=False, raw=False, prepare=True, preponly=False, refresh=True, build=None): @@ -3735,3 +3765,25 @@ def force_exit(exitvalue=0): sys.stdout.flush() sys.stderr.flush() os._exit(exitvalue) + + +YAML_LINT_CONFIG = {'extends': 'default', + 'rules': {'document-start': 'disable', + 'line-length': 'disable', + 'truthy': 'disable'}} + + +def run_yamllint(path, indent=0): + + try: + import yamllint.config + import yamllint.linter + except ImportError: + return '' + + result = [] + with open(path, 'r', encoding='utf-8') as f: + problems = yamllint.linter.run(f, yamllint.config.YamlLintConfig(json.dumps(YAML_LINT_CONFIG))) + for problem in problems: + result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message) + return '\n'.join(result) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index ab7d422c..b39dd873 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -574,6 +574,9 @@ def main(): common.setup_global_opts(parser) parser.add_argument("-f", "--format", action="store_true", default=False, help=_("Also warn about formatting issues, like rewritemeta -l")) + parser.add_argument('--force-yamllint', action="store_true", default=False, + help=_("When linting the entire repository yamllint is disabled by default. " + "This option forces yamllint regardless.")) parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID")) metadata.add_metadata_arguments(parser) options = parser.parse_args() @@ -600,6 +603,29 @@ def main(): if app.Disabled: continue + # only run yamllint when linting individual apps. + if len(options.appid) > 0 or options.force_yamllint: + + # run yamllint on app metadata + ymlpath = os.path.join('metadata', appid + '.yml') + if os.path.isfile(ymlpath): + yamllintresult = common.run_yamllint(ymlpath) + if yamllintresult != '': + print(yamllintresult) + + # run yamllint on srclib metadata + srclibs = set() + for build in app.builds: + for srclib in build.srclibs: + srclibs.add(srclib) + for srclib in srclibs: + name, ref, number, subdir = common.parse_srclib_spec(srclib) + srclibpath = os.path.join('srclibs', name + '.yml') + if os.path.isfile(srclibpath): + yamllintresult = common.run_yamllint(srclibpath) + if yamllintresult != '': + print(yamllintresult) + app_check_funcs = [ check_app_field_types, check_regexes, diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 3f79db43..99764ac4 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -747,7 +747,7 @@ def parse_txt_srclib(metadatapath): return thisinfo -def parse_yml_srclib(metadatapath): +def parse_yaml_srclib(metadatapath): thisinfo = {'RepoType': '', 'Repo': '', @@ -765,9 +765,11 @@ def parse_yml_srclib(metadatapath): data = yaml.load(f, Loader=SafeLoader) except yaml.error.YAMLError as e: warn_or_exception(_("Invalid srclib metadata: could not " - "parse '{file}'" - .format(file=metadatapath)), - e) + "parse '{file}'") + .format(file=metadatapath) + '\n' + + fdroidserver.common.run_yamllint(metadatapath, + indent=4), + cause=e) return thisinfo for key in data.keys(): @@ -820,7 +822,7 @@ def read_srclibs(): for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))): srclibname = os.path.basename(metadatapath[:-4]) - srclibs[srclibname] = parse_yml_srclib(metadatapath) + srclibs[srclibname] = parse_yaml_srclib(metadatapath) def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): @@ -1102,7 +1104,14 @@ def parse_json_metadata(mf, app): def parse_yaml_metadata(mf, app): - yamldata = yaml.load(mf, Loader=SafeLoader) + try: + yamldata = yaml.load(mf, Loader=SafeLoader) + except yaml.YAMLError as e: + warn_or_exception(_("could not parse '{path}'") + .format(path=mf.name) + '\n' + + fdroidserver.common.run_yamllint(mf.name, + indent=4), + cause=e) deprecated_in_yaml = ['Provides'] diff --git a/setup.py b/setup.py index 40abdb19..bade76bb 100755 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ setup(name='fdroidserver', 'qrcode', 'ruamel.yaml >= 0.15', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', + 'yamllint', ], classifiers=[ 'Development Status :: 4 - Beta', diff --git a/tests/common.TestCase b/tests/common.TestCase index 2ee7852f..93fbedd3 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -34,7 +34,7 @@ import fdroidserver.signindex import fdroidserver.common import fdroidserver.metadata from testcommon import TmpCwd -from fdroidserver.exception import FDroidException, VCSException +from fdroidserver.exception import FDroidException, VCSException, MetaDataException class CommonTest(unittest.TestCase): @@ -1014,6 +1014,22 @@ class CommonTest(unittest.TestCase): subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths) self.assertEqual(subdirs[f], subdir) + def test_parse_srclib_spec_good(self): + self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'), + ('osmand-external-skia', 'android/oreo', None, None)) + self.assertEqual(fdroidserver.common.parse_srclib_spec('1:appcompat@v7'), + ('appcompat', 'v7', '1', None)) + self.assertEqual(fdroidserver.common.parse_srclib_spec('1:Support/v7/appcompat@android-4.4_r1.1'), + ('Support', 'android-4.4_r1.1', '1', 'v7/appcompat')) + + def test_parse_srclib_spec_bad(self): + with self.assertRaises(MetaDataException): + self.assertEqual(fdroidserver.common.parse_srclib_spec(None)) + with self.assertRaises(MetaDataException): + self.assertEqual(fdroidserver.common.parse_srclib_spec('no-ref')) + with self.assertRaises(MetaDataException): + self.assertEqual(fdroidserver.common.parse_srclib_spec('@multi@at-signs@')) + def test_bad_urls(self): for url in ('asdf', 'file://thing.git', @@ -1298,6 +1314,32 @@ class CommonTest(unittest.TestCase): dfm.assert_called_once_with('srclib/ACRA') self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA')) + def test_run_yamllint_wellformed(self): + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('wellformed.yml', 'w') as f: + f.write(textwrap.dedent('''\ + yaml: + file: + - for + - test + purposeses: true + ''')) + result = fdroidserver.common.run_yamllint('wellformed.yml') + self.assertEqual(result, '') + + def test_run_yamllint_malformed(self): + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with open('malformed.yml', 'w') as f: + f.write(textwrap.dedent('''\ + yaml: + - that + fails + - test + ''')) + result = fdroidserver.common.run_yamllint('malformed.yml') + self.assertIsNotNone(result) + self.assertNotEqual(result, '') + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index dedbdd3f..d52cd991 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -652,7 +652,7 @@ class MetadataTest(unittest.TestCase): "'android.library=true\\ntarget=android-19' > project.properties"}, srclib) - def test_parse_yml_srclib_unknown_key(self): + def test_parse_yaml_srclib_unknown_key(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('test.yml', 'w', encoding='utf-8') as f: @@ -665,17 +665,17 @@ class MetadataTest(unittest.TestCase): "Invalid srclib metadata: " "unknown key 'Evil' in " "'test.yml'"): - fdroidserver.metadata.parse_yml_srclib('test.yml') + fdroidserver.metadata.parse_yaml_srclib('test.yml') - def test_parse_yml_srclib_does_not_exists(self): + def test_parse_yaml_srclib_does_not_exists(self): fdroidserver.metadata.warnings_action = 'error' with self.assertRaisesRegex(MetaDataException, "Invalid scrlib metadata: " "'non/existent-test-srclib.yml' " "does not exist"): - fdroidserver.metadata.parse_yml_srclib('non/existent-test-srclib.yml') + fdroidserver.metadata.parse_yaml_srclib('non/existent-test-srclib.yml') - def test_parse_yml_srclib_simple(self): + def test_parse_yaml_srclib_simple(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('simple.yml', 'w', encoding='utf-8') as f: @@ -684,14 +684,14 @@ class MetadataTest(unittest.TestCase): RepoType: git Repo: https://git.host/repo.git ''')) - srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml') + srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml') self.assertDictEqual({'Repo': 'https://git.host/repo.git', 'RepoType': 'git', 'Subdir': None, 'Prepare': None}, srclib) - def test_parse_yml_srclib_simple_with_blanks(self): + def test_parse_yaml_srclib_simple_with_blanks(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('simple.yml', 'w', encoding='utf-8') as f: @@ -706,14 +706,14 @@ class MetadataTest(unittest.TestCase): Prepare: ''')) - srclib = fdroidserver.metadata.parse_yml_srclib('simple.yml') + srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml') self.assertDictEqual({'Repo': 'https://git.host/repo.git', 'RepoType': 'git', 'Subdir': [''], 'Prepare': ''}, srclib) - def test_parse_yml_srclib_Changelog_cketti(self): + def test_parse_yaml_srclib_Changelog_cketti(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with open('Changelog-cketti.yml', 'w', encoding='utf-8') as f: @@ -724,7 +724,7 @@ class MetadataTest(unittest.TestCase): Subdir: library,ckChangeLog/src/main Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties" ''')) - srclib = fdroidserver.metadata.parse_yml_srclib('Changelog-cketti.yml') + srclib = fdroidserver.metadata.parse_yaml_srclib('Changelog-cketti.yml') self.assertDictEqual(srclib, {'Repo': 'https://github.com/cketti/ckChangeLog', 'RepoType': 'git',