diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py
index 3d5b700f..30084695 100644
--- a/fdroidserver/scanner.py
+++ b/fdroidserver/scanner.py
@@ -16,8 +16,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+import json
import os
import re
+import sys
import traceback
from argparse import ArgumentParser
import logging
@@ -31,6 +33,8 @@ from .exception import BuildException, VCSException
config = None
options = None
+json_per_build = None
+
def get_gradle_compile_commands(build):
compileCommands = ['compile',
@@ -141,10 +145,14 @@ def scan_source(build_dir, build=metadata.Build()):
def ignoreproblem(what, path_in_build_dir):
logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
+ if json_per_build is not None:
+ json_per_build['infos'].append([what, path_in_build_dir])
return 0
def removeproblem(what, path_in_build_dir, filepath):
logging.info('Removing %s at %s' % (what, path_in_build_dir))
+ if json_per_build is not None:
+ json_per_build['infos'].append([what, path_in_build_dir])
os.remove(filepath)
return 0
@@ -152,13 +160,18 @@ def scan_source(build_dir, build=metadata.Build()):
if toignore(path_in_build_dir):
return
logging.warn('Found %s at %s' % (what, path_in_build_dir))
+ if json_per_build is not None:
+ json_per_build['warnings'].append([what, path_in_build_dir])
def handleproblem(what, path_in_build_dir, filepath):
if toignore(path_in_build_dir):
return ignoreproblem(what, path_in_build_dir)
if todelete(path_in_build_dir):
return removeproblem(what, path_in_build_dir, filepath)
- logging.error('Found %s at %s' % (what, path_in_build_dir))
+ if options.json:
+ json_per_build['errors'].append([what, path_in_build_dir])
+ if not options.json or options.verbose:
+ logging.error('Found %s at %s' % (what, path_in_build_dir))
return 1
def is_executable(path):
@@ -249,7 +262,8 @@ def scan_source(build_dir, build=metadata.Build()):
for i, line in enumerate(lines):
if is_used_by_gradle(line):
for name in suspects_found(line):
- count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
+ count += handleproblem("usual suspect \'%s\'" % (name),
+ path_in_build_dir, filepath)
noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]
joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
for m in gradle_mavenrepo.finditer(joined):
@@ -280,7 +294,7 @@ def scan_source(build_dir, build=metadata.Build()):
def main():
- global config, options
+ global config, options, json_per_build
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
@@ -288,10 +302,19 @@ def main():
parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
parser.add_argument("-f", "--force", action="store_true", default=False,
help=_("Force scan of disabled apps and builds."))
+ parser.add_argument("--json", action="store_true", default=False,
+ help=_("Output JSON to stdout."))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
+ json_output = dict()
+ if options.json:
+ if options.verbose:
+ logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+ else:
+ logging.getLogger().setLevel(logging.ERROR)
+
config = common.read_config(options)
# Read all app and srclib metadata
@@ -309,8 +332,11 @@ def main():
for appid, app in apps.items():
+ json_per_appid = dict()
+
if app.Disabled and not options.force:
logging.info(_("Skipping {appid}: disabled").format(appid=appid))
+ json_per_appid = json_per_appid['infos'].append('Skipping: disabled')
continue
try:
@@ -321,20 +347,23 @@ def main():
if app.builds:
logging.info(_("Processing {appid}").format(appid=appid))
+ # Set up vcs interface and make sure we have the latest code...
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
else:
logging.info(_("{appid}: no builds specified, running on current source state")
.format(appid=appid))
+ json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+ json_per_appid['current-source-state'] = json_per_build
count = scan_source(build_dir)
if count > 0:
logging.warn(_('Scanner found {count} problems in {appid}:')
.format(count=count, appid=appid))
probcount += count
- continue
-
- # Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
+ app.builds = []
for build in app.builds:
+ json_per_build = {'errors': [], 'warnings': [], 'infos': []}
+ json_per_appid[build.versionCode] = json_per_build
if build.disable and not options.force:
logging.info("...skipping version %s - %s" % (
@@ -365,8 +394,16 @@ def main():
appid, traceback.format_exc()))
probcount += 1
+ for k, v in json_per_appid.items():
+ if len(v['errors']) or len(v['warnings']) or len(v['infos']):
+ json_output[appid] = json_per_appid
+ break
+
logging.info(_("Finished"))
- print(_("%d problems found") % probcount)
+ if options.json:
+ print(json.dumps(json_output))
+ else:
+ print(_("%d problems found") % probcount)
if __name__ == "__main__":
diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase
index cbaa9de2..c6c1232c 100755
--- a/tests/scanner.TestCase
+++ b/tests/scanner.TestCase
@@ -6,7 +6,10 @@ import logging
import optparse
import os
import sys
+import tempfile
+import textwrap
import unittest
+from unittest import mock
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -14,6 +17,7 @@ print('localmodule: ' + localmodule)
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
+import fdroidserver.build
import fdroidserver.common
import fdroidserver.metadata
import fdroidserver.scanner
@@ -24,8 +28,14 @@ class ScannerTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.INFO)
self.basedir = os.path.join(localmodule, 'tests')
+ self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
+ if not os.path.exists(self.tmpdir):
+ os.makedirs(self.tmpdir)
+ os.chdir(self.basedir)
def test_scan_source_files(self):
+ fdroidserver.scanner.options = type('', (), {})()
+ fdroidserver.scanner.options.json = False
source_files = os.path.join(self.basedir, 'source-files')
projects = {
'cn.wildfirechat.chat': 4,
@@ -43,6 +53,63 @@ class ScannerTest(unittest.TestCase):
self.assertEqual(should, fatal_problems,
"%s should have %d errors!" % (d, should))
+ def test_build_local_scanner(self):
+ """`fdroid build` calls scanner functions, test them here"""
+ testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+ os.chdir(testdir)
+
+ config = dict()
+ fdroidserver.common.fill_config_defaults(config)
+ fdroidserver.common.config = config
+ fdroidserver.build.options = mock.Mock()
+ fdroidserver.build.options.json = False
+ fdroidserver.build.options.notarball = True
+ fdroidserver.build.options.skipscan = False
+ fdroidserver.scanner.options = fdroidserver.build.options
+
+ app = fdroidserver.metadata.App()
+ app.id = 'mocked.app.id'
+ build = fdroidserver.metadata.Build()
+ build.commit = '1.0'
+ build.output = app.id + '.apk'
+ build.scanignore = ['baz.so']
+ build.versionCode = '1'
+ build.versionName = '1.0'
+ vcs = mock.Mock()
+
+ for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'):
+ with open(f, 'w') as fp:
+ fp.write('placeholder')
+ self.assertTrue(os.path.exists(f))
+
+ with open('build.xml', 'w') as fp:
+ fp.write(textwrap.dedent(
+ """
+
+
+
+ """))
+
+ def make_fake_apk(output, build):
+ with open(build.output, 'w') as fp:
+ fp.write('APK PLACEHOLDER')
+ return output
+
+ with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk):
+ with mock.patch('fdroidserver.common.get_native_code', return_value='x86'):
+ with mock.patch('fdroidserver.common.get_apk_id',
+ return_value=(app.id, build.versionCode, build.versionName)):
+ with mock.patch('fdroidserver.common.is_apk_and_debuggable', return_value=False):
+ fdroidserver.build.build_local(
+ app, build, vcs,
+ build_dir=testdir, output_dir=testdir,
+ log_dir=None, srclib_dir=None, extlib_dir=None, tmp_dir=None,
+ force=False, onserver=False, refresh=False
+ )
+ self.assertTrue(os.path.exists('baz.so'))
+ self.assertTrue(os.path.exists('foo.aar'))
+ self.assertFalse(os.path.exists('gradle-wrapper.jar'))
+
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))