kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-14 19:10:11 +01:00
Code Activity

Merge branch 'scanner_refactoring' into 'master'

Browse Source 
Scanner refactoring

See merge request !330
This commit is contained in:
Hans-Christoph Steiner 2017-09-11 15:47:57 +00:00
commit 9461dd0f3d
1 changed files with 44 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
fdroidserver/scanner.py
View File

@ -97,42 +97,42 @@ def scan_source(build_dir, build):
scanignore_worked = set() scanignore_worked = set()
scandelete_worked = set() scandelete_worked = set()
def toignore(fd): def toignore(path_in_build_dir):
for k, paths in scanignore.items(): for k, paths in scanignore.items():
for p in paths: for p in paths:
if fd.startswith(p): if path_in_build_dir.startswith(p):
scanignore_worked.add(k) scanignore_worked.add(k)
return True return True
return False return False
def todelete(fd): def todelete(path_in_build_dir):
for k, paths in scandelete.items(): for k, paths in scandelete.items():
for p in paths: for p in paths:
if fd.startswith(p): if path_in_build_dir.startswith(p):
scandelete_worked.add(k) scandelete_worked.add(k)
return True return True
return False return False
def ignoreproblem(what, fd): def ignoreproblem(what, path_in_build_dir):
logging.info('Ignoring %s at %s' % (what, fd)) logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
return 0 return 0
def removeproblem(what, fd, fp): def removeproblem(what, path_in_build_dir, filepath):
logging.info('Removing %s at %s' % (what, fd)) logging.info('Removing %s at %s' % (what, path_in_build_dir))
os.remove(fp) os.remove(filepath)
return 0 return 0
def warnproblem(what, fd): def warnproblem(what, path_in_build_dir):
if toignore(fd): if toignore(path_in_build_dir):
return return
logging.warn('Found %s at %s' % (what, fd)) logging.warn('Found %s at %s' % (what, path_in_build_dir))
def handleproblem(what, fd, fp): def handleproblem(what, path_in_build_dir, filepath):
if toignore(fd): if toignore(path_in_build_dir):
return ignoreproblem(what, fd) return ignoreproblem(what, path_in_build_dir)
if todelete(fd): if todelete(path_in_build_dir):
return removeproblem(what, fd, fp) return removeproblem(what, path_in_build_dir, filepath)
logging.error('Found %s at %s' % (what, fd)) logging.error('Found %s at %s' % (what, path_in_build_dir))
return 1 return 1
def is_executable(path): def is_executable(path):
@ -165,73 +165,73 @@ def scan_source(build_dir, build):
return any(command.match(line) for command in gradle_compile_commands) return any(command.match(line) for command in gradle_compile_commands)
# Iterate through all files in the source code # Iterate through all files in the source code
for r, d, f in os.walk(build_dir, topdown=True): for dirpath, dirnames, filenames in os.walk(build_dir, topdown=True):
# It's topdown, so checking the basename is enough # It's topdown, so checking the basename is enough
for ignoredir in ('.hg', '.git', '.svn', '.bzr'): for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
if ignoredir in d: if ignoredir in dirnames:
d.remove(ignoredir) dirnames.remove(ignoredir)
for curfile in f: for curfile in filenames:
if curfile in ['.DS_Store']: if curfile in ['.DS_Store']:
continue continue
# Path (relative) to the file # Path (relative) to the file
fp = os.path.join(r, curfile) filepath = os.path.join(dirpath, curfile)
if os.path.islink(fp): if os.path.islink(filepath):
continue continue
fd = fp[len(build_dir) + 1:] path_in_build_dir = os.path.relpath(filepath, build_dir)
_, ext = common.get_extension(fd) _, ext = common.get_extension(path_in_build_dir)
if ext == 'so': if ext == 'so':
count += handleproblem('shared library', fd, fp) count += handleproblem('shared library', path_in_build_dir, filepath)
elif ext == 'a': elif ext == 'a':
count += handleproblem('static library', fd, fp) count += handleproblem('static library', path_in_build_dir, filepath)
elif ext == 'class': elif ext == 'class':
count += handleproblem('Java compiled class', fd, fp) count += handleproblem('Java compiled class', path_in_build_dir, filepath)
elif ext == 'apk': elif ext == 'apk':
removeproblem('APK file', fd, fp) removeproblem('APK file', path_in_build_dir, filepath)
elif ext == 'jar': elif ext == 'jar':
for name in suspects_found(curfile): for name in suspects_found(curfile):
count += handleproblem('usual supect \'%s\'' % name, fd, fp) count += handleproblem('usual supect \'%s\'' % name, path_in_build_dir, filepath)
warnproblem('JAR file', fd) warnproblem('JAR file', path_in_build_dir)
elif ext == 'java': elif ext == 'java':
if not os.path.isfile(fp): if not os.path.isfile(filepath):
continue continue
with open(fp, 'r', encoding='utf8', errors='replace') as f: with open(filepath, 'r', encoding='utf8', errors='replace') as f:
for line in f: for line in f:
if 'DexClassLoader' in line: if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp) count += handleproblem('DexClassLoader', path_in_build_dir, filepath)
break break
elif ext == 'gradle': elif ext == 'gradle':
if not os.path.isfile(fp): if not os.path.isfile(filepath):
continue continue
with open(fp, 'r', encoding='utf8', errors='replace') as f: with open(filepath, 'r', encoding='utf8', errors='replace') as f:
lines = f.readlines() lines = f.readlines()
for i, line in enumerate(lines): for i, line in enumerate(lines):
if is_used_by_gradle(line): if is_used_by_gradle(line):
for name in suspects_found(line): for name in suspects_found(line):
count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), fd, fp) count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)] noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines)) joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
for m in gradle_mavenrepo.finditer(joined): for m in gradle_mavenrepo.finditer(joined):
url = m.group(2) url = m.group(2)
if not any(r.match(url) for r in allowed_repos): if not any(r.match(url) for r in allowed_repos):
count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp) count += handleproblem('unknown maven repo \'%s\'' % url, path_in_build_dir, filepath)
elif ext in ['', 'bin', 'out', 'exe']: elif ext in ['', 'bin', 'out', 'exe']:
if is_binary(fp): if is_binary(filepath):
count += handleproblem('binary', fd, fp) count += handleproblem('binary', path_in_build_dir, filepath)
elif is_executable(fp): elif is_executable(filepath):
if is_binary(fp) and not safe_path(fd): if is_binary(filepath) and not safe_path(path_in_build_dir):
warnproblem('possible binary', fd) warnproblem('possible binary', path_in_build_dir)
for p in scanignore: for p in scanignore:
if p not in scanignore_worked: if p not in scanignore_worked: