From 67332d83a5bca2bbeeff1466148857cb69089a4d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 May 2020 16:08:56 +0200 Subject: [PATCH] scanner: add --json option for outputting machine readable results * makes per-build entries in per-app entries * `fdroid scanner --json --verbose` will output logging messages to stderr * removed " at line N" from one message to make them uniform keys * this will be used in issuebot This is a second attempt with tests for how `fdroid build` calls the scanner functions. closes #771. It was previously merged in !748 then reverted in 68c072c72e2d9c75b5220ea1e95d1773e8ae2723 --- fdroidserver/scanner.py | 53 +++++++++++++++++++++++++++----- tests/scanner.TestCase | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 8 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 3d5b700f..30084695 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -16,8 +16,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json import os import re +import sys import traceback from argparse import ArgumentParser import logging @@ -31,6 +33,8 @@ from .exception import BuildException, VCSException config = None options = None +json_per_build = None + def get_gradle_compile_commands(build): compileCommands = ['compile', @@ -141,10 +145,14 @@ def scan_source(build_dir, build=metadata.Build()): def ignoreproblem(what, path_in_build_dir): logging.info('Ignoring %s at %s' % (what, path_in_build_dir)) + if json_per_build is not None: + json_per_build['infos'].append([what, path_in_build_dir]) return 0 def removeproblem(what, path_in_build_dir, filepath): logging.info('Removing %s at %s' % (what, path_in_build_dir)) + if json_per_build is not None: + json_per_build['infos'].append([what, path_in_build_dir]) os.remove(filepath) return 0 @@ -152,13 +160,18 @@ def scan_source(build_dir, build=metadata.Build()): if toignore(path_in_build_dir): return logging.warn('Found %s at %s' % (what, path_in_build_dir)) + if json_per_build is not None: + json_per_build['warnings'].append([what, path_in_build_dir]) def handleproblem(what, path_in_build_dir, filepath): if toignore(path_in_build_dir): return ignoreproblem(what, path_in_build_dir) if todelete(path_in_build_dir): return removeproblem(what, path_in_build_dir, filepath) - logging.error('Found %s at %s' % (what, path_in_build_dir)) + if options.json: + json_per_build['errors'].append([what, path_in_build_dir]) + if not options.json or options.verbose: + logging.error('Found %s at %s' % (what, path_in_build_dir)) return 1 def is_executable(path): @@ -249,7 +262,8 @@ def scan_source(build_dir, build=metadata.Build()): for i, line in enumerate(lines): if is_used_by_gradle(line): for name in suspects_found(line): - count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath) + count += handleproblem("usual suspect \'%s\'" % (name), + path_in_build_dir, filepath) noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)] joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines)) for m in gradle_mavenrepo.finditer(joined): @@ -280,7 +294,7 @@ def scan_source(build_dir, build=metadata.Build()): def main(): - global config, options + global config, options, json_per_build # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") @@ -288,10 +302,19 @@ def main(): parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) parser.add_argument("-f", "--force", action="store_true", default=False, help=_("Force scan of disabled apps and builds.")) + parser.add_argument("--json", action="store_true", default=False, + help=_("Output JSON to stdout.")) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W + json_output = dict() + if options.json: + if options.verbose: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + else: + logging.getLogger().setLevel(logging.ERROR) + config = common.read_config(options) # Read all app and srclib metadata @@ -309,8 +332,11 @@ def main(): for appid, app in apps.items(): + json_per_appid = dict() + if app.Disabled and not options.force: logging.info(_("Skipping {appid}: disabled").format(appid=appid)) + json_per_appid = json_per_appid['infos'].append('Skipping: disabled') continue try: @@ -321,20 +347,23 @@ def main(): if app.builds: logging.info(_("Processing {appid}").format(appid=appid)) + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) else: logging.info(_("{appid}: no builds specified, running on current source state") .format(appid=appid)) + json_per_build = {'errors': [], 'warnings': [], 'infos': []} + json_per_appid['current-source-state'] = json_per_build count = scan_source(build_dir) if count > 0: logging.warn(_('Scanner found {count} problems in {appid}:') .format(count=count, appid=appid)) probcount += count - continue - - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + app.builds = [] for build in app.builds: + json_per_build = {'errors': [], 'warnings': [], 'infos': []} + json_per_appid[build.versionCode] = json_per_build if build.disable and not options.force: logging.info("...skipping version %s - %s" % ( @@ -365,8 +394,16 @@ def main(): appid, traceback.format_exc())) probcount += 1 + for k, v in json_per_appid.items(): + if len(v['errors']) or len(v['warnings']) or len(v['infos']): + json_output[appid] = json_per_appid + break + logging.info(_("Finished")) - print(_("%d problems found") % probcount) + if options.json: + print(json.dumps(json_output)) + else: + print(_("%d problems found") % probcount) if __name__ == "__main__": diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index cbaa9de2..c6c1232c 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -6,7 +6,10 @@ import logging import optparse import os import sys +import tempfile +import textwrap import unittest +from unittest import mock localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) @@ -14,6 +17,7 @@ print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) +import fdroidserver.build import fdroidserver.common import fdroidserver.metadata import fdroidserver.scanner @@ -24,8 +28,14 @@ class ScannerTest(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.INFO) self.basedir = os.path.join(localmodule, 'tests') + self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + os.chdir(self.basedir) def test_scan_source_files(self): + fdroidserver.scanner.options = type('', (), {})() + fdroidserver.scanner.options.json = False source_files = os.path.join(self.basedir, 'source-files') projects = { 'cn.wildfirechat.chat': 4, @@ -43,6 +53,63 @@ class ScannerTest(unittest.TestCase): self.assertEqual(should, fatal_problems, "%s should have %d errors!" % (d, should)) + def test_build_local_scanner(self): + """`fdroid build` calls scanner functions, test them here""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.build.options = mock.Mock() + fdroidserver.build.options.json = False + fdroidserver.build.options.notarball = True + fdroidserver.build.options.skipscan = False + fdroidserver.scanner.options = fdroidserver.build.options + + app = fdroidserver.metadata.App() + app.id = 'mocked.app.id' + build = fdroidserver.metadata.Build() + build.commit = '1.0' + build.output = app.id + '.apk' + build.scanignore = ['baz.so'] + build.versionCode = '1' + build.versionName = '1.0' + vcs = mock.Mock() + + for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'): + with open(f, 'w') as fp: + fp.write('placeholder') + self.assertTrue(os.path.exists(f)) + + with open('build.xml', 'w') as fp: + fp.write(textwrap.dedent( + """ + + + + """)) + + def make_fake_apk(output, build): + with open(build.output, 'w') as fp: + fp.write('APK PLACEHOLDER') + return output + + with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk): + with mock.patch('fdroidserver.common.get_native_code', return_value='x86'): + with mock.patch('fdroidserver.common.get_apk_id', + return_value=(app.id, build.versionCode, build.versionName)): + with mock.patch('fdroidserver.common.is_apk_and_debuggable', return_value=False): + fdroidserver.build.build_local( + app, build, vcs, + build_dir=testdir, output_dir=testdir, + log_dir=None, srclib_dir=None, extlib_dir=None, tmp_dir=None, + force=False, onserver=False, refresh=False + ) + self.assertTrue(os.path.exists('baz.so')) + self.assertTrue(os.path.exists('foo.aar')) + self.assertFalse(os.path.exists('gradle-wrapper.jar')) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))