mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
Merge branch 'scanner-json' into 'master'
`fdroid scanner --json` for machine readable output See merge request fdroid/fdroidserver!748
This commit is contained in:
Hans-Christoph Steiner
commit
1be8788f5f
@ -763,6 +763,9 @@ def parse_yaml_srclib(metadatapath):
|
||||
with open(metadatapath, "r", encoding="utf-8") as f:
|
||||
try:
|
||||
data = yaml.load(f, Loader=SafeLoader)
|
||||
if type(data) is not dict:
|
||||
raise yaml.error.YAMLError(_('{file} is blank or corrupt!')
|
||||
.format(file=metadatapath))
|
||||
except yaml.error.YAMLError as e:
|
||||
warn_or_exception(_("Invalid srclib metadata: could not "
|
||||
"parse '{file}'")
|
||||
|
||||
@ -16,8 +16,10 @@
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from argparse import ArgumentParser
|
||||
import logging
|
||||
@ -31,6 +33,8 @@ from .exception import BuildException, VCSException
|
||||
config = None
|
||||
options = None
|
||||
|
||||
json_per_build = None
|
||||
|
||||
|
||||
def get_gradle_compile_commands(build):
|
||||
compileCommands = ['compile',
|
||||
@ -141,10 +145,12 @@ def scan_source(build_dir, build=metadata.Build()):
|
||||
|
||||
def ignoreproblem(what, path_in_build_dir):
|
||||
logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
|
||||
json_per_build['infos'].append([what, path_in_build_dir])
|
||||
return 0
|
||||
|
||||
def removeproblem(what, path_in_build_dir, filepath):
|
||||
logging.info('Removing %s at %s' % (what, path_in_build_dir))
|
||||
json_per_build['infos'].append([what, path_in_build_dir])
|
||||
os.remove(filepath)
|
||||
return 0
|
||||
|
||||
@ -152,13 +158,17 @@ def scan_source(build_dir, build=metadata.Build()):
|
||||
if toignore(path_in_build_dir):
|
||||
return
|
||||
logging.warn('Found %s at %s' % (what, path_in_build_dir))
|
||||
json_per_build['warnings'].append([what, path_in_build_dir])
|
||||
|
||||
def handleproblem(what, path_in_build_dir, filepath):
|
||||
if toignore(path_in_build_dir):
|
||||
return ignoreproblem(what, path_in_build_dir)
|
||||
if todelete(path_in_build_dir):
|
||||
return removeproblem(what, path_in_build_dir, filepath)
|
||||
logging.error('Found %s at %s' % (what, path_in_build_dir))
|
||||
if options.json:
|
||||
json_per_build['errors'].append([what, path_in_build_dir])
|
||||
if not options.json or options.verbose:
|
||||
logging.error('Found %s at %s' % (what, path_in_build_dir))
|
||||
return 1
|
||||
|
||||
def is_executable(path):
|
||||
@ -249,7 +259,8 @@ def scan_source(build_dir, build=metadata.Build()):
|
||||
for i, line in enumerate(lines):
|
||||
if is_used_by_gradle(line):
|
||||
for name in suspects_found(line):
|
||||
count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
|
||||
count += handleproblem("usual suspect \'%s\'" % (name),
|
||||
path_in_build_dir, filepath)
|
||||
noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]
|
||||
joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
|
||||
for m in gradle_mavenrepo.finditer(joined):
|
||||
@ -280,7 +291,7 @@ def scan_source(build_dir, build=metadata.Build()):
|
||||
|
||||
def main():
|
||||
|
||||
global config, options
|
||||
global config, options, json_per_build
|
||||
|
||||
# Parse command line...
|
||||
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
|
||||
@ -288,10 +299,19 @@ def main():
|
||||
parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
|
||||
parser.add_argument("-f", "--force", action="store_true", default=False,
|
||||
help=_("Force scan of disabled apps and builds."))
|
||||
parser.add_argument("--json", action="store_true", default=False,
|
||||
help=_("Output JSON to stdout."))
|
||||
metadata.add_metadata_arguments(parser)
|
||||
options = parser.parse_args()
|
||||
metadata.warnings_action = options.W
|
||||
|
||||
json_output = dict()
|
||||
if options.json:
|
||||
if options.verbose:
|
||||
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
||||
else:
|
||||
logging.getLogger().setLevel(logging.ERROR)
|
||||
|
||||
config = common.read_config(options)
|
||||
|
||||
# Read all app and srclib metadata
|
||||
@ -309,8 +329,11 @@ def main():
|
||||
|
||||
for appid, app in apps.items():
|
||||
|
||||
json_per_appid = dict()
|
||||
|
||||
if app.Disabled and not options.force:
|
||||
logging.info(_("Skipping {appid}: disabled").format(appid=appid))
|
||||
json_per_appid = json_per_appid['infos'].append('Skipping: disabled')
|
||||
continue
|
||||
|
||||
try:
|
||||
@ -321,20 +344,23 @@ def main():
|
||||
|
||||
if app.builds:
|
||||
logging.info(_("Processing {appid}").format(appid=appid))
|
||||
# Set up vcs interface and make sure we have the latest code...
|
||||
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
||||
else:
|
||||
logging.info(_("{appid}: no builds specified, running on current source state")
|
||||
.format(appid=appid))
|
||||
json_per_build = {'errors': [], 'warnings': [], 'infos': []}
|
||||
json_per_appid['current-source-state'] = json_per_build
|
||||
count = scan_source(build_dir)
|
||||
if count > 0:
|
||||
logging.warn(_('Scanner found {count} problems in {appid}:')
|
||||
.format(count=count, appid=appid))
|
||||
probcount += count
|
||||
continue
|
||||
|
||||
# Set up vcs interface and make sure we have the latest code...
|
||||
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
||||
app.builds = []
|
||||
|
||||
for build in app.builds:
|
||||
json_per_build = {'errors': [], 'warnings': [], 'infos': []}
|
||||
json_per_appid[build.versionCode] = json_per_build
|
||||
|
||||
if build.disable and not options.force:
|
||||
logging.info("...skipping version %s - %s" % (
|
||||
@ -365,8 +391,16 @@ def main():
|
||||
appid, traceback.format_exc()))
|
||||
probcount += 1
|
||||
|
||||
for k, v in json_per_appid.items():
|
||||
if len(v['errors']) or len(v['warnings']) or len(v['infos']):
|
||||
json_output[appid] = json_per_appid
|
||||
break
|
||||
|
||||
logging.info(_("Finished"))
|
||||
print(_("%d problems found") % probcount)
|
||||
if options.json:
|
||||
print(json.dumps(json_output))
|
||||
else:
|
||||
print(_("%d problems found") % probcount)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -227,6 +227,19 @@ class MetadataTest(unittest.TestCase):
|
||||
with self.assertRaises(MetaDataException):
|
||||
fdroidserver.metadata.parse_yaml_metadata(mf, {})
|
||||
|
||||
def test_parse_yaml_srclib_corrupt_file(self):
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
srclibfile = os.path.join(testdir, 'srclib', 'mock.yml')
|
||||
os.mkdir(os.path.dirname(srclibfile))
|
||||
with open(srclibfile, 'w') as fp:
|
||||
fp.write(textwrap.dedent("""
|
||||
- RepoType: git
|
||||
- Repo: https://github.com/realm/realm-js.git
|
||||
"""))
|
||||
with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
|
||||
with self.assertRaises(MetaDataException):
|
||||
fdroidserver.metadata.parse_yaml_srclib(srclibfile)
|
||||
|
||||
def test_write_yaml_with_placeholder_values(self):
|
||||
mf = io.StringIO()
|
||||
|
||||
|
||||
@ -26,6 +26,8 @@ class ScannerTest(unittest.TestCase):
|
||||
self.basedir = os.path.join(localmodule, 'tests')
|
||||
|
||||
def test_scan_source_files(self):
|
||||
fdroidserver.scanner.options = type('', (), {})()
|
||||
fdroidserver.scanner.options.json = False
|
||||
source_files = os.path.join(self.basedir, 'source-files')
|
||||
projects = {
|
||||
'cn.wildfirechat.chat': 4,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user