scanner: add --json option for outputting machine readable results

* makes per-build entries in per-app entries
* `fdroid scanner --json --verbose` will output logging messages to stderr
* removed " at line N" from one message to make them uniform keys
* this will be used in issuebot

This is a second attempt with tests for how `fdroid build` calls the
scanner functions. closes #771.  It was previously merged in !748 then
reverted in 68c072c72e

fdroidserver/scanner.py

@@ -16,8 +16,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
 import os
 import re
+import sys
 import traceback
 from argparse import ArgumentParser
 import logging
@@ -31,6 +33,8 @@ from .exception import BuildException, VCSException
 config = None
 options = None
 
+json_per_build = None
+
 
 def get_gradle_compile_commands(build):
     compileCommands = ['compile',
@@ -141,10 +145,14 @@ def scan_source(build_dir, build=metadata.Build()):
 
     def ignoreproblem(what, path_in_build_dir):
         logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
+        if json_per_build is not None:
+            json_per_build['infos'].append([what, path_in_build_dir])
         return 0
 
     def removeproblem(what, path_in_build_dir, filepath):
         logging.info('Removing %s at %s' % (what, path_in_build_dir))
+        if json_per_build is not None:
+            json_per_build['infos'].append([what, path_in_build_dir])
         os.remove(filepath)
         return 0
 
@@ -152,13 +160,18 @@ def scan_source(build_dir, build=metadata.Build()):
         if toignore(path_in_build_dir):
             return
         logging.warn('Found %s at %s' % (what, path_in_build_dir))
+        if json_per_build is not None:
+            json_per_build['warnings'].append([what, path_in_build_dir])
 
     def handleproblem(what, path_in_build_dir, filepath):
         if toignore(path_in_build_dir):
             return ignoreproblem(what, path_in_build_dir)
         if todelete(path_in_build_dir):
             return removeproblem(what, path_in_build_dir, filepath)
-        logging.error('Found %s at %s' % (what, path_in_build_dir))
+        if options.json:
+            json_per_build['errors'].append([what, path_in_build_dir])
+        if not options.json or options.verbose:
+            logging.error('Found %s at %s' % (what, path_in_build_dir))
         return 1
 
     def is_executable(path):
@@ -249,7 +262,8 @@ def scan_source(build_dir, build=metadata.Build()):
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):
                         for name in suspects_found(line):
-                            count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
+                            count += handleproblem("usual suspect \'%s\'" % (name),
+                                                   path_in_build_dir, filepath)
                 noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]
                 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
                 for m in gradle_mavenrepo.finditer(joined):
@@ -280,7 +294,7 @@ def scan_source(build_dir, build=metadata.Build()):
 
 def main():
 
-    global config, options
+    global config, options, json_per_build
 
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
@@ -288,10 +302,19 @@ def main():
     parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     parser.add_argument("-f", "--force", action="store_true", default=False,
                         help=_("Force scan of disabled apps and builds."))
+    parser.add_argument("--json", action="store_true", default=False,
+                        help=_("Output JSON to stdout."))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
 
+    json_output = dict()
+    if options.json:
+        if options.verbose:
+            logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+        else:
+            logging.getLogger().setLevel(logging.ERROR)
+
     config = common.read_config(options)
 
     # Read all app and srclib metadata
@@ -309,8 +332,11 @@ def main():
 
     for appid, app in apps.items():
 
+        json_per_appid = dict()
+
         if app.Disabled and not options.force:
             logging.info(_("Skipping {appid}: disabled").format(appid=appid))
+            json_per_appid = json_per_appid['infos'].append('Skipping: disabled')
             continue
 
         try:
@@ -321,20 +347,23 @@ def main():
 
             if app.builds:
                 logging.info(_("Processing {appid}").format(appid=appid))
+                # Set up vcs interface and make sure we have the latest code...
+                vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
             else:
                 logging.info(_("{appid}: no builds specified, running on current source state")
                              .format(appid=appid))
+                json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+                json_per_appid['current-source-state'] = json_per_build
                 count = scan_source(build_dir)
                 if count > 0:
                     logging.warn(_('Scanner found {count} problems in {appid}:')
                                  .format(count=count, appid=appid))
                     probcount += count
-                continue
+                app.builds = []
-
-            # Set up vcs interface and make sure we have the latest code...
-            vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
 
             for build in app.builds:
+                json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+                json_per_appid[build.versionCode] = json_per_build
 
                 if build.disable and not options.force:
                     logging.info("...skipping version %s - %s" % (
@@ -365,8 +394,16 @@ def main():
                 appid, traceback.format_exc()))
             probcount += 1
 
+        for k, v in json_per_appid.items():
+            if len(v['errors']) or len(v['warnings']) or len(v['infos']):
+                json_output[appid] = json_per_appid
+                break
+
     logging.info(_("Finished"))
-    print(_("%d problems found") % probcount)
+    if options.json:
+        print(json.dumps(json_output))
+    else:
+        print(_("%d problems found") % probcount)
 
 
 if __name__ == "__main__":

tests/scanner.TestCase

@@ -6,7 +6,10 @@ import logging
 import optparse
 import os
 import sys
+import tempfile
+import textwrap
 import unittest
+from unittest import mock
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -14,6 +17,7 @@ print('localmodule: ' + localmodule)
 if localmodule not in sys.path:
     sys.path.insert(0, localmodule)
 
+import fdroidserver.build
 import fdroidserver.common
 import fdroidserver.metadata
 import fdroidserver.scanner
@@ -24,8 +28,14 @@ class ScannerTest(unittest.TestCase):
     def setUp(self):
         logging.basicConfig(level=logging.INFO)
         self.basedir = os.path.join(localmodule, 'tests')
+        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
+        if not os.path.exists(self.tmpdir):
+            os.makedirs(self.tmpdir)
+        os.chdir(self.basedir)
 
     def test_scan_source_files(self):
+        fdroidserver.scanner.options = type('', (), {})()
+        fdroidserver.scanner.options.json = False
         source_files = os.path.join(self.basedir, 'source-files')
         projects = {
             'cn.wildfirechat.chat': 4,
@@ -43,6 +53,63 @@ class ScannerTest(unittest.TestCase):
             self.assertEqual(should, fatal_problems,
                              "%s should have %d errors!" % (d, should))
 
+    def test_build_local_scanner(self):
+        """`fdroid build` calls scanner functions, test them here"""
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.build.options = mock.Mock()
+        fdroidserver.build.options.json = False
+        fdroidserver.build.options.notarball = True
+        fdroidserver.build.options.skipscan = False
+        fdroidserver.scanner.options = fdroidserver.build.options
+
+        app = fdroidserver.metadata.App()
+        app.id = 'mocked.app.id'
+        build = fdroidserver.metadata.Build()
+        build.commit = '1.0'
+        build.output = app.id + '.apk'
+        build.scanignore = ['baz.so']
+        build.versionCode = '1'
+        build.versionName = '1.0'
+        vcs = mock.Mock()
+
+        for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'):
+            with open(f, 'w') as fp:
+                fp.write('placeholder')
+            self.assertTrue(os.path.exists(f))
+
+        with open('build.xml', 'w') as fp:
+            fp.write(textwrap.dedent(
+                """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+                <project basedir="." default="clean" name="mockapp">
+                <target name="release"/>
+                <target name="clean"/>
+                </project>"""))
+
+        def make_fake_apk(output, build):
+            with open(build.output, 'w') as fp:
+                fp.write('APK PLACEHOLDER')
+            return output
+
+        with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk):
+            with mock.patch('fdroidserver.common.get_native_code', return_value='x86'):
+                with mock.patch('fdroidserver.common.get_apk_id',
+                                return_value=(app.id, build.versionCode, build.versionName)):
+                    with mock.patch('fdroidserver.common.is_apk_and_debuggable', return_value=False):
+                        fdroidserver.build.build_local(
+                            app, build, vcs,
+                            build_dir=testdir, output_dir=testdir,
+                            log_dir=None, srclib_dir=None, extlib_dir=None, tmp_dir=None,
+                            force=False, onserver=False, refresh=False
+                        )
+        self.assertTrue(os.path.exists('baz.so'))
+        self.assertTrue(os.path.exists('foo.aar'))
+        self.assertFalse(os.path.exists('gradle-wrapper.jar'))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))