kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00
Code Activity

Merge branch 'lock' into 'master'

Browse Source 
scanner: error on dependency files without lock file

Closes #1200

See merge request fdroid/fdroidserver!1504
This commit is contained in:
Hans-Christoph Steiner 2024-08-31 13:13:02 +00:00
commit 3a1bbb54aa
8 changed files with 233 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

346
fdroidserver/scanner.py
View File

@ -16,29 +16,26 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import imghdr
import itertools
import json
import logging
import os import os
import re import re
import sys import sys
import json
import imghdr
import logging
import zipfile
import itertools
import traceback import traceback
import urllib.parse import urllib.parse
import urllib.request import urllib.request
import zipfile
from argparse import ArgumentParser from argparse import ArgumentParser
from tempfile import TemporaryDirectory
from pathlib import Path
from datetime import datetime, timedelta
from dataclasses import dataclass, field, fields from dataclasses import dataclass, field, fields
from datetime import datetime, timedelta
from enum import IntEnum from enum import IntEnum
from pathlib import Path
from tempfile import TemporaryDirectory
from . import _ from . import _, common, metadata, scanner
from . import common from .exception import BuildException, ConfigurationException, VCSException
from . import metadata
from .exception import BuildException, VCSException, ConfigurationException
from . import scanner
@dataclass @dataclass
@ -48,9 +45,16 @@ class MessageStore:
errors: list = field(default_factory=list) errors: list = field(default_factory=list)
MAVEN_URL_REGEX = re.compile(r"""\smaven\s*(?:{.*?(?:setUrl|url)|\(\s*(?:url)?)\s*=?\s*(?:uri|URI|Uri\.create)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""", MAVEN_URL_REGEX = re.compile(
re.DOTALL) r"""\smaven\s*(?:{.*?(?:setUrl|url)|\(\s*(?:url)?)\s*=?\s*(?:uri|URI|Uri\.create)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""",
re.DOTALL,
)
DEPFILE = {
"Cargo.toml": ["Cargo.lock"],
"pubspec.yaml": ["pubspec.lock"],
"package.json": ["package.lock", "yarn.lock", "pnpm-lock.yaml"],
}
SCANNER_CACHE_VERSION = 1 SCANNER_CACHE_VERSION = 1
@ -60,21 +64,25 @@ class ExitCode(IntEnum):
def get_gradle_compile_commands(build): def get_gradle_compile_commands(build):
compileCommands = ['api', compileCommands = [
'apk', 'api',
'classpath', 'apk',
'compile', 'classpath',
'compileOnly', 'compile',
'id', 'compileOnly',
'implementation', 'id',
'provided', 'implementation',
'runtimeOnly'] 'provided',
'runtimeOnly',
]
buildTypes = ['', 'release'] buildTypes = ['', 'release']
flavors = [''] flavors = ['']
if build.gradle and build.gradle != ['yes']: if build.gradle and build.gradle != ['yes']:
flavors += build.gradle flavors += build.gradle
commands = [''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)] commands = [
''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)
]
return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands] return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]
@ -115,7 +123,9 @@ def get_embedded_classes(apkfile, depth=0):
["dexdump", '{}/{}'.format(tmp_dir, info.filename)], ["dexdump", '{}/{}'.format(tmp_dir, info.filename)],
output=False, output=False,
) )
classes = classes.union(set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output))) classes = classes.union(
set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output))
)
except zipfile.BadZipFile as ex: except zipfile.BadZipFile as ex:
return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)} return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)}
@ -191,9 +201,11 @@ class SignatureDataController:
raise SignatureDataMalformedException() from e raise SignatureDataMalformedException() from e
delta = (last_updated + self.cache_duration) - scanner._datetime_now() delta = (last_updated + self.cache_duration) - scanner._datetime_now()
if delta > timedelta(seconds=0): if delta > timedelta(seconds=0):
logging.debug(_('next {name} cache update due in {time}').format( logging.debug(
name=self.filename, time=delta _('next {name} cache update due in {time}').format(
)) name=self.filename, time=delta
)
)
else: else:
raise SignatureDataOutdatedException() raise SignatureDataOutdatedException()
@ -202,7 +214,9 @@ class SignatureDataController:
self.fetch_signatures_from_web() self.fetch_signatures_from_web()
self.write_to_cache() self.write_to_cache()
except Exception as e: except Exception as e:
raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e raise Exception(
_("downloading scanner signatures from '{}' failed").format(self.url)
) from e
def load(self): def load(self):
try: try:
@ -215,10 +229,17 @@ class SignatureDataController:
except (SignatureDataOutdatedException, SignatureDataNoDefaultsException): except (SignatureDataOutdatedException, SignatureDataNoDefaultsException):
self.fetch_signatures_from_web() self.fetch_signatures_from_web()
self.write_to_cache() self.write_to_cache()
except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e: except (
logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format( SignatureDataMalformedException,
clear='rm -r {}'.format(common.get_config()['cachedir_scanner']) SignatureDataVersionMismatchException,
)) ) as e:
logging.critical(
_(
"scanner cache is malformed! You can clear it with: '{clear}'"
).format(
clear='rm -r {}'.format(common.get_config()['cachedir_scanner'])
)
)
raise e raise e
def load_from_defaults(self): def load_from_defaults(self):
@ -244,7 +265,13 @@ class SignatureDataController:
Right now this function does just a basic key sanitation. Right now this function does just a basic key sanitation.
""" """
self.check_data_version() self.check_data_version()
valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration'] valid_keys = [
'timestamp',
'last_updated',
'version',
'signatures',
'cache_duration',
]
for k in list(self.data.keys()): for k in list(self.data.keys()):
if k not in valid_keys: if k not in valid_keys:
@ -266,7 +293,11 @@ class SignatureDataController:
class ExodusSignatureDataController(SignatureDataController): class ExodusSignatureDataController(SignatureDataController):
def __init__(self): def __init__(self):
super().__init__('Exodus signatures', 'exodus.json', 'https://reports.exodus-privacy.eu.org/api/trackers') super().__init__(
'Exodus signatures',
'exodus.json',
'https://reports.exodus-privacy.eu.org/api/trackers',
)
self.cache_duration = timedelta(days=1) # refresh exodus cache after one day self.cache_duration = timedelta(days=1) # refresh exodus cache after one day
self.has_trackers_json_key = True self.has_trackers_json_key = True
@ -294,11 +325,11 @@ class ExodusSignatureDataController(SignatureDataController):
# exodus also provides network signatures, unused atm. # exodus also provides network signatures, unused atm.
# "network_signatures": [tracker["network_signature"]], # "network_signatures": [tracker["network_signature"]],
"AntiFeatures": ["Tracking"], # TODO "AntiFeatures": ["Tracking"], # TODO
"license": "NonFree" # We assume all trackers in exodus "license": "NonFree", # We assume all trackers in exodus
# are non-free, although free # are non-free, although free
# trackers like piwik, acra, # trackers like piwik, acra,
# etc. might be listed by exodus # etc. might be listed by exodus
# too. # too.
} }
self.set_data(data) self.set_data(data)
@ -315,9 +346,7 @@ class EtipSignatureDataController(ExodusSignatureDataController):
class SUSSDataController(SignatureDataController): class SUSSDataController(SignatureDataController):
def __init__(self): def __init__(self):
super().__init__( super().__init__(
'SUSS', 'SUSS', 'suss.json', 'https://fdroid.gitlab.io/fdroid-suss/suss.json'
'suss.json',
'https://fdroid.gitlab.io/fdroid-suss/suss.json'
) )
def load_from_defaults(self): def load_from_defaults(self):
@ -332,7 +361,9 @@ class ScannerTool:
self.scanner_data_lookup() self.scanner_data_lookup()
options = common.get_options() options = common.get_options()
options_refresh_scanner = hasattr(options, "refresh_scanner") and options.refresh_scanner options_refresh_scanner = (
hasattr(options, "refresh_scanner") and options.refresh_scanner
)
if options_refresh_scanner or common.get_config().get('refresh_scanner'): if options_refresh_scanner or common.get_config().get('refresh_scanner'):
self.refresh() self.refresh()
@ -342,8 +373,9 @@ class ScannerTool:
def scanner_data_lookup(self): def scanner_data_lookup(self):
sigsources = common.get_config().get('scanner_signature_sources', []) sigsources = common.get_config().get('scanner_signature_sources', [])
logging.debug( logging.debug(
"scanner is configured to use signature data from: '{}'" "scanner is configured to use signature data from: '{}'".format(
.format("', '".join(sigsources)) "', '".join(sigsources)
)
) )
self.sdcs = [] self.sdcs = []
for i, source_url in enumerate(sigsources): for i, source_url in enumerate(sigsources):
@ -361,11 +393,13 @@ class ScannerTool:
"Has to be a valid HTTPS-URL or match a predefined " "Has to be a valid HTTPS-URL or match a predefined "
"constants: 'suss', 'exodus'".format(source_url) "constants: 'suss', 'exodus'".format(source_url)
) )
self.sdcs.append(SignatureDataController( self.sdcs.append(
source_url, SignatureDataController(
'{}_{}'.format(i, os.path.basename(u.path)), source_url,
source_url, '{}_{}'.format(i, os.path.basename(u.path)),
)) source_url,
)
)
def load(self): def load(self):
for sdc in self.sdcs: for sdc in self.sdcs:
@ -381,13 +415,21 @@ class ScannerTool:
for sdc in self.sdcs: for sdc in self.sdcs:
for signame, sigdef in sdc.data.get('signatures', {}).items(): for signame, sigdef in sdc.data.get('signatures', {}).items():
for sig in sigdef.get('code_signatures', []): for sig in sigdef.get('code_signatures', []):
self.regexs['err_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) self.regexs['err_code_signatures'][sig] = re.compile(
'.*' + sig, re.IGNORECASE
)
for sig in sigdef.get('gradle_signatures', []): for sig in sigdef.get('gradle_signatures', []):
self.regexs['err_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) self.regexs['err_gradle_signatures'][sig] = re.compile(
'.*' + sig, re.IGNORECASE
)
for sig in sigdef.get('warn_code_signatures', []): for sig in sigdef.get('warn_code_signatures', []):
self.regexs['warn_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) self.regexs['warn_code_signatures'][sig] = re.compile(
'.*' + sig, re.IGNORECASE
)
for sig in sigdef.get('warn_gradle_signatures', []): for sig in sigdef.get('warn_gradle_signatures', []):
self.regexs['warn_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE) self.regexs['warn_gradle_signatures'][sig] = re.compile(
'.*' + sig, re.IGNORECASE
)
def refresh(self): def refresh(self):
for sdc in self.sdcs: for sdc in self.sdcs:
@ -430,9 +472,17 @@ def scan_binary(apkfile):
logging.debug("Problem: found class '%s'" % classname) logging.debug("Problem: found class '%s'" % classname)
problems += 1 problems += 1
if warnings: if warnings:
logging.warning(_("Found {count} warnings in {filename}").format(count=warnings, filename=apkfile)) logging.warning(
_("Found {count} warnings in {filename}").format(
count=warnings, filename=apkfile
)
)
if problems: if problems:
logging.critical(_("Found {count} problems in {filename}").format(count=problems, filename=apkfile)) logging.critical(
_("Found {count} problems in {filename}").format(
count=problems, filename=apkfile
)
)
return problems return problems
@ -453,29 +503,33 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
if r.match(s): if r.match(s):
yield n yield n
allowed_repos = [re.compile(r'^https://' + re.escape(repo) + r'/*') for repo in [ allowed_repos = [
'repo1.maven.org/maven2', # mavenCentral() re.compile(r'^https://' + re.escape(repo) + r'/*')
'jcenter.bintray.com', # jcenter() for repo in [
'jitpack.io', 'repo1.maven.org/maven2', # mavenCentral()
'www.jitpack.io', 'jcenter.bintray.com', # jcenter()
'repo.maven.apache.org/maven2', 'jitpack.io',
'oss.jfrog.org/artifactory/oss-snapshot-local', 'www.jitpack.io',
'oss.sonatype.org/content/repositories/snapshots', 'repo.maven.apache.org/maven2',
'oss.sonatype.org/content/repositories/releases', 'oss.jfrog.org/artifactory/oss-snapshot-local',
'oss.sonatype.org/content/groups/public', 'oss.sonatype.org/content/repositories/snapshots',
'oss.sonatype.org/service/local/staging/deploy/maven2', 'oss.sonatype.org/content/repositories/releases',
's01.oss.sonatype.org/content/repositories/snapshots', 'oss.sonatype.org/content/groups/public',
's01.oss.sonatype.org/content/repositories/releases', 'oss.sonatype.org/service/local/staging/deploy/maven2',
's01.oss.sonatype.org/content/groups/public', 's01.oss.sonatype.org/content/repositories/snapshots',
's01.oss.sonatype.org/service/local/staging/deploy/maven2', 's01.oss.sonatype.org/content/repositories/releases',
'clojars.org/repo', # Clojure free software libs 's01.oss.sonatype.org/content/groups/public',
'repo.clojars.org', # Clojure free software libs 's01.oss.sonatype.org/service/local/staging/deploy/maven2',
's3.amazonaws.com/repo.commonsware.com', # CommonsWare 'clojars.org/repo', # Clojure free software libs
'plugins.gradle.org/m2', # Gradle plugin repo 'repo.clojars.org', # Clojure free software libs
'maven.google.com', # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven 's3.amazonaws.com/repo.commonsware.com', # CommonsWare
'plugins.gradle.org/m2', # Gradle plugin repo
'maven.google.com', # google()
] ]
] + [re.compile(r'^file://' + re.escape(repo) + r'/*') for repo in [ ] + [
'/usr/share/maven-repo', # local repo on Debian installs re.compile(r'^file://' + re.escape(repo) + r'/*')
for repo in [
'/usr/share/maven-repo', # local repo on Debian installs
] ]
] ]
@ -515,7 +569,7 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
------- -------
0 as we explicitly ignore the file, so don't count an error 0 as we explicitly ignore the file, so don't count an error
""" """
msg = ('Ignoring %s at %s' % (what, path_in_build_dir)) msg = 'Ignoring %s at %s' % (what, path_in_build_dir)
logging.info(msg) logging.info(msg)
if json_per_build is not None: if json_per_build is not None:
json_per_build.infos.append([msg, path_in_build_dir]) json_per_build.infos.append([msg, path_in_build_dir])
@ -537,7 +591,7 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
------- -------
0 as we deleted the offending file 0 as we deleted the offending file
""" """
msg = ('Removing %s at %s' % (what, path_in_build_dir)) msg = 'Removing %s at %s' % (what, path_in_build_dir)
logging.info(msg) logging.info(msg)
if json_per_build is not None: if json_per_build is not None:
json_per_build.infos.append([msg, path_in_build_dir]) json_per_build.infos.append([msg, path_in_build_dir])
@ -598,14 +652,16 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
return warnproblem(what, path_in_build_dir, json_per_build) return warnproblem(what, path_in_build_dir, json_per_build)
if options and 'json' in vars(options) and options.json: if options and 'json' in vars(options) and options.json:
json_per_build.errors.append([what, path_in_build_dir]) json_per_build.errors.append([what, path_in_build_dir])
if options and (options.verbose or not ('json' in vars(options) and options.json)): if options and (
options.verbose or not ('json' in vars(options) and options.json)
):
logging.error('Found %s at %s' % (what, path_in_build_dir)) logging.error('Found %s at %s' % (what, path_in_build_dir))
return 1 return 1
def is_executable(path): def is_executable(path):
return os.path.exists(path) and os.access(path, os.X_OK) return os.path.exists(path) and os.access(path, os.X_OK)
textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f}) textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f}) # fmt: skip
def is_binary(path): def is_binary(path):
d = None d = None
@ -614,9 +670,11 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
return bool(d.translate(None, textchars)) return bool(d.translate(None, textchars))
# False positives patterns for files that are binary and executable. # False positives patterns for files that are binary and executable.
safe_paths = [re.compile(r) for r in [ safe_paths = [
r".*/drawable[^/]*/.*\.png$", # png drawables re.compile(r)
r".*/mipmap[^/]*/.*\.png$", # png mipmaps for r in [
r".*/drawable[^/]*/.*\.png$", # png drawables
r".*/mipmap[^/]*/.*\.png$", # png mipmaps
] ]
] ]
@ -637,14 +695,12 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
# Iterate through all files in the source code # Iterate through all files in the source code
for root, dirs, files in os.walk(build_dir, topdown=True): for root, dirs, files in os.walk(build_dir, topdown=True):
# It's topdown, so checking the basename is enough # It's topdown, so checking the basename is enough
for ignoredir in ('.hg', '.git', '.svn', '.bzr'): for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
if ignoredir in dirs: if ignoredir in dirs:
dirs.remove(ignoredir) dirs.remove(ignoredir)
for curfile in files: for curfile in files:
if curfile in ['.DS_Store']: if curfile in ['.DS_Store']:
continue continue
@ -733,13 +789,17 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
if is_used_by_gradle(line): if is_used_by_gradle(line):
for name in suspects_found(line): for name in suspects_found(line):
count += handleproblem( count += handleproblem(
"usual suspect \'%s\'" % (name), "usual suspect '%s'" % (name),
path_in_build_dir, path_in_build_dir,
filepath, filepath,
json_per_build, json_per_build,
) )
noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)] noncomment_lines = [
no_comments = re.sub(r'/\*.*?\*/', '', ''.join(noncomment_lines), flags=re.DOTALL) line for line in lines if not common.gradle_comment.match(line)
]
no_comments = re.sub(
r'/\*.*?\*/', '', ''.join(noncomment_lines), flags=re.DOTALL
)
for url in MAVEN_URL_REGEX.findall(no_comments): for url in MAVEN_URL_REGEX.findall(no_comments):
if not any(r.match(url) for r in allowed_repos): if not any(r.match(url) for r in allowed_repos):
count += handleproblem( count += handleproblem(
@ -755,8 +815,22 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
'binary', path_in_build_dir, filepath, json_per_build 'binary', path_in_build_dir, filepath, json_per_build
) )
elif curfile in DEPFILE:
for lockfile in DEPFILE[curfile]:
if os.path.isfile(os.path.join(root, lockfile)):
break
else:
count += handleproblem(
_('dependency file without lock'),
path_in_build_dir,
filepath,
json_per_build,
)
elif is_executable(filepath): elif is_executable(filepath):
if is_binary(filepath) and not (safe_path(path_in_build_dir) or is_image_file(filepath)): if is_binary(filepath) and not (
safe_path(path_in_build_dir) or is_image_file(filepath)
):
warnproblem( warnproblem(
_('executable binary, possibly code'), _('executable binary, possibly code'),
path_in_build_dir, path_in_build_dir,
@ -781,15 +855,36 @@ def main():
usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]" usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]"
) )
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) parser.add_argument(
parser.add_argument("-f", "--force", action="store_true", default=False, "appid",
help=_("Force scan of disabled apps and builds.")) nargs='*',
parser.add_argument("--json", action="store_true", default=False, help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
help=_("Output JSON to stdout.")) )
parser.add_argument("-r", "--refresh", dest="refresh_scanner", action="store_true", default=False, parser.add_argument(
help=_("fetch the latest version of signatures from the web")) "-f",
parser.add_argument("-e", "--exit-code", action="store_true", default=False, "--force",
help=_("Exit with a non-zero code if problems were found")) action="store_true",
default=False,
help=_("Force scan of disabled apps and builds."),
)
parser.add_argument(
"--json", action="store_true", default=False, help=_("Output JSON to stdout.")
)
parser.add_argument(
"-r",
"--refresh",
dest="refresh_scanner",
action="store_true",
default=False,
help=_("fetch the latest version of signatures from the web"),
)
parser.add_argument(
"-e",
"--exit-code",
action="store_true",
default=False,
help=_("Exit with a non-zero code if problems were found"),
)
metadata.add_metadata_arguments(parser) metadata.add_metadata_arguments(parser)
options = common.parse_args(parser) options = common.parse_args(parser)
metadata.warnings_action = options.W metadata.warnings_action = options.W
@ -840,7 +935,6 @@ def main():
extlib_dir = os.path.join(build_dir, 'extlib') extlib_dir = os.path.join(build_dir, 'extlib')
for appid, app in apps.items(): for appid, app in apps.items():
json_per_appid = dict() json_per_appid = dict()
if app.Disabled and not options.force: if app.Disabled and not options.force:
@ -861,14 +955,20 @@ def main():
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir) vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
else: else:
logging.info(_("{appid}: no builds specified, running on current source state") logging.info(
.format(appid=appid)) _(
"{appid}: no builds specified, running on current source state"
).format(appid=appid)
)
json_per_build = MessageStore() json_per_build = MessageStore()
json_per_appid['current-source-state'] = json_per_build json_per_appid['current-source-state'] = json_per_build
count = scan_source(build_dir, json_per_build=json_per_build) count = scan_source(build_dir, json_per_build=json_per_build)
if count > 0: if count > 0:
logging.warning(_('Scanner found {count} problems in {appid}:') logging.warning(
.format(count=count, appid=appid)) _('Scanner found {count} problems in {appid}:').format(
count=count, appid=appid
)
)
probcount += count probcount += count
app['Builds'] = [] app['Builds'] = []
@ -877,32 +977,42 @@ def main():
json_per_appid[build.versionCode] = json_per_build json_per_appid[build.versionCode] = json_per_build
if build.disable and not options.force: if build.disable and not options.force:
logging.info("...skipping version %s - %s" % ( logging.info(
build.versionName, build.get('disable', build.commit[1:]))) "...skipping version %s - %s"
% (build.versionName, build.get('disable', build.commit[1:]))
)
continue continue
logging.info("...scanning version " + build.versionName) logging.info("...scanning version " + build.versionName)
# Prepare the source code... # Prepare the source code...
common.prepare_source(vcs, app, build, common.prepare_source(
build_dir, srclib_dir, vcs, app, build, build_dir, srclib_dir, extlib_dir, False
extlib_dir, False) )
count = scan_source(build_dir, build, json_per_build=json_per_build) count = scan_source(build_dir, build, json_per_build=json_per_build)
if count > 0: if count > 0:
logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:') logging.warning(
.format(count=count, appid=appid, versionCode=build.versionCode)) _(
'Scanner found {count} problems in {appid}:{versionCode}:'
).format(
count=count, appid=appid, versionCode=build.versionCode
)
)
probcount += count probcount += count
except BuildException as be: except BuildException as be:
logging.warning('Could not scan app %s due to BuildException: %s' % ( logging.warning(
appid, be)) 'Could not scan app %s due to BuildException: %s' % (appid, be)
)
probcount += 1 probcount += 1
except VCSException as vcse: except VCSException as vcse:
logging.warning('VCS error while scanning app %s: %s' % (appid, vcse)) logging.warning('VCS error while scanning app %s: %s' % (appid, vcse))
probcount += 1 probcount += 1
except Exception: except Exception:
logging.warning('Could not scan app %s due to unknown error: %s' % ( logging.warning(
appid, traceback.format_exc())) 'Could not scan app %s due to unknown error: %s'
% (appid, traceback.format_exc())
)
probcount += 1 probcount += 1
for k, v in json_per_appid.items(): for k, v in json_per_appid.items():

4
pyproject.toml
View File

@ -43,7 +43,6 @@ force-exclude = '''(
| fdroidserver/metadata\.py | fdroidserver/metadata\.py
| fdroidserver/nightly\.py | fdroidserver/nightly\.py
| fdroidserver/publish\.py | fdroidserver/publish\.py
| fdroidserver/scanner\.py
| fdroidserver/update\.py | fdroidserver/update\.py
| fdroidserver/vmtools\.py | fdroidserver/vmtools\.py
| locale/pick-complete-translations\.py | locale/pick-complete-translations\.py
@ -141,3 +140,6 @@ max-nested-blocks = 5
[tool.pylint.format] [tool.pylint.format]
# Maximum number of characters on a single line. # Maximum number of characters on a single line.
max-line-length = 88 max-line-length = 88
[tool.ruff.format]
quote-style = "preserve"

3
tests/scanner.TestCase
View File

@ -60,7 +60,8 @@ class ScannerTest(unittest.TestCase):
'org.mozilla.rocket': 2, 'org.mozilla.rocket': 2,
'org.tasks': 2, 'org.tasks': 2,
'realm': 1, 'realm': 1,
'se.manyver': 2, 'se.manyver': 3,
'lockfile.test': 1,
} }
for d in glob.glob(os.path.join(source_files, '*')): for d in glob.glob(os.path.join(source_files, '*')):
build = fdroidserver.metadata.Build() build = fdroidserver.metadata.Build()

0
tests/source-files/lockfile.test/Cargo.lock generated Normal file
View File

0
tests/source-files/lockfile.test/Cargo.toml Normal file
View File

0
tests/source-files/lockfile.test/package.json Normal file
View File

0
tests/source-files/lockfile.test/pubspec.yaml Normal file
View File

0
tests/source-files/lockfile.test/yarn.lock Normal file
View File