mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-11 15:13:27 +02:00
Move scan_source into scanner.py
Not really a common.py thing.
This commit is contained in:
parent
925fbee3b9
commit
120be4334d
@ -35,6 +35,7 @@ import logging
|
|||||||
|
|
||||||
import common
|
import common
|
||||||
import metadata
|
import metadata
|
||||||
|
import scanner
|
||||||
from common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
|
from common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -555,7 +556,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
|||||||
else:
|
else:
|
||||||
# Scan before building...
|
# Scan before building...
|
||||||
logging.info("Scanning source for common problems...")
|
logging.info("Scanning source for common problems...")
|
||||||
count = common.scan_source(build_dir, root_dir, thisbuild)
|
count = scanner.scan_source(build_dir, root_dir, thisbuild)
|
||||||
if count > 0:
|
if count > 0:
|
||||||
if force:
|
if force:
|
||||||
logging.warn('Scanner found %d problems' % count)
|
logging.warn('Scanner found %d problems' % count)
|
||||||
|
@ -1439,219 +1439,6 @@ def getpaths(build_dir, build, field):
|
|||||||
return paths
|
return paths
|
||||||
|
|
||||||
|
|
||||||
def init_mime_type():
|
|
||||||
'''
|
|
||||||
There are two incompatible versions of the 'magic' module, one
|
|
||||||
that comes as part of libmagic, which is what Debian includes as
|
|
||||||
python-magic, then another called python-magic that is a separate
|
|
||||||
project that wraps libmagic. The second is 'magic' on pypi, so
|
|
||||||
both need to be supported. Then on platforms where libmagic is
|
|
||||||
not easily included, e.g. OSX and Windows, fallback to the
|
|
||||||
built-in 'mimetypes' module so this will work without
|
|
||||||
libmagic. Hence this function with the following hacks:
|
|
||||||
'''
|
|
||||||
|
|
||||||
init_path = ''
|
|
||||||
method = ''
|
|
||||||
ms = None
|
|
||||||
|
|
||||||
def mime_from_file(path):
|
|
||||||
try:
|
|
||||||
return magic.from_file(path, mime=True)
|
|
||||||
except UnicodeError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
def mime_file(path):
|
|
||||||
try:
|
|
||||||
return ms.file(path)
|
|
||||||
except UnicodeError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
def mime_guess_type(path):
|
|
||||||
return mimetypes.guess_type(path, strict=False)
|
|
||||||
|
|
||||||
try:
|
|
||||||
import magic
|
|
||||||
try:
|
|
||||||
ms = magic.open(magic.MIME_TYPE)
|
|
||||||
ms.load()
|
|
||||||
magic.from_file(init_path, mime=True)
|
|
||||||
method = 'from_file'
|
|
||||||
except AttributeError:
|
|
||||||
ms.file(init_path)
|
|
||||||
method = 'file'
|
|
||||||
except ImportError:
|
|
||||||
import mimetypes
|
|
||||||
mimetypes.init()
|
|
||||||
method = 'guess_type'
|
|
||||||
|
|
||||||
logging.info("Using magic method " + method)
|
|
||||||
if method == 'from_file':
|
|
||||||
return mime_from_file
|
|
||||||
if method == 'file':
|
|
||||||
return mime_file
|
|
||||||
if method == 'guess_type':
|
|
||||||
return mime_guess_type
|
|
||||||
|
|
||||||
logging.critical("unknown magic method!")
|
|
||||||
|
|
||||||
|
|
||||||
# Scan the source code in the given directory (and all subdirectories)
|
|
||||||
# and return the number of fatal problems encountered
|
|
||||||
def scan_source(build_dir, root_dir, thisbuild):
|
|
||||||
|
|
||||||
count = 0
|
|
||||||
|
|
||||||
# Common known non-free blobs (always lower case):
|
|
||||||
usual_suspects = [
|
|
||||||
re.compile(r'.*flurryagent', re.IGNORECASE),
|
|
||||||
re.compile(r'.*paypal.*mpl', re.IGNORECASE),
|
|
||||||
re.compile(r'.*google.*analytics', re.IGNORECASE),
|
|
||||||
re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
|
|
||||||
re.compile(r'.*google.*ad.*view', re.IGNORECASE),
|
|
||||||
re.compile(r'.*google.*admob', re.IGNORECASE),
|
|
||||||
re.compile(r'.*google.*play.*services', re.IGNORECASE),
|
|
||||||
re.compile(r'.*crittercism', re.IGNORECASE),
|
|
||||||
re.compile(r'.*heyzap', re.IGNORECASE),
|
|
||||||
re.compile(r'.*jpct.*ae', re.IGNORECASE),
|
|
||||||
re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
|
|
||||||
re.compile(r'.*bugsense', re.IGNORECASE),
|
|
||||||
re.compile(r'.*crashlytics', re.IGNORECASE),
|
|
||||||
re.compile(r'.*ouya.*sdk', re.IGNORECASE),
|
|
||||||
re.compile(r'.*libspen23', re.IGNORECASE),
|
|
||||||
]
|
|
||||||
|
|
||||||
scanignore = getpaths(build_dir, thisbuild, 'scanignore')
|
|
||||||
scandelete = getpaths(build_dir, thisbuild, 'scandelete')
|
|
||||||
|
|
||||||
scanignore_worked = set()
|
|
||||||
scandelete_worked = set()
|
|
||||||
|
|
||||||
def toignore(fd):
|
|
||||||
for p in scanignore:
|
|
||||||
if fd.startswith(p):
|
|
||||||
scanignore_worked.add(p)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def todelete(fd):
|
|
||||||
for p in scandelete:
|
|
||||||
if fd.startswith(p):
|
|
||||||
scandelete_worked.add(p)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def ignoreproblem(what, fd, fp):
|
|
||||||
logging.info('Ignoring %s at %s' % (what, fd))
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def removeproblem(what, fd, fp):
|
|
||||||
logging.info('Removing %s at %s' % (what, fd))
|
|
||||||
os.remove(fp)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def warnproblem(what, fd):
|
|
||||||
logging.warn('Found %s at %s' % (what, fd))
|
|
||||||
|
|
||||||
def handleproblem(what, fd, fp):
|
|
||||||
if toignore(fd):
|
|
||||||
return ignoreproblem(what, fd, fp)
|
|
||||||
if todelete(fd):
|
|
||||||
return removeproblem(what, fd, fp)
|
|
||||||
logging.error('Found %s at %s' % (what, fd))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
get_mime_type = init_mime_type()
|
|
||||||
|
|
||||||
# Iterate through all files in the source code
|
|
||||||
for r, d, f in os.walk(build_dir, topdown=True):
|
|
||||||
|
|
||||||
# It's topdown, so checking the basename is enough
|
|
||||||
for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
|
|
||||||
if ignoredir in d:
|
|
||||||
d.remove(ignoredir)
|
|
||||||
|
|
||||||
for curfile in f:
|
|
||||||
|
|
||||||
# Path (relative) to the file
|
|
||||||
fp = os.path.join(r, curfile)
|
|
||||||
fd = fp[len(build_dir) + 1:]
|
|
||||||
|
|
||||||
mime = get_mime_type(fp)
|
|
||||||
|
|
||||||
if mime == 'application/x-sharedlib':
|
|
||||||
count += handleproblem('shared library', fd, fp)
|
|
||||||
|
|
||||||
elif mime == 'application/x-archive':
|
|
||||||
count += handleproblem('static library', fd, fp)
|
|
||||||
|
|
||||||
elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
|
|
||||||
count += handleproblem('binary executable', fd, fp)
|
|
||||||
|
|
||||||
elif mime == 'application/x-java-applet':
|
|
||||||
count += handleproblem('Java compiled class', fd, fp)
|
|
||||||
|
|
||||||
elif mime in (
|
|
||||||
'application/jar',
|
|
||||||
'application/zip',
|
|
||||||
'application/java-archive',
|
|
||||||
'application/octet-stream',
|
|
||||||
'binary', ):
|
|
||||||
|
|
||||||
if has_extension(fp, 'apk'):
|
|
||||||
removeproblem('APK file', fd, fp)
|
|
||||||
|
|
||||||
elif has_extension(fp, 'jar'):
|
|
||||||
|
|
||||||
if any(suspect.match(curfile) for suspect in usual_suspects):
|
|
||||||
count += handleproblem('usual supect', fd, fp)
|
|
||||||
else:
|
|
||||||
warnproblem('JAR file', fd)
|
|
||||||
|
|
||||||
elif has_extension(fp, 'zip'):
|
|
||||||
warnproblem('ZIP file', fd)
|
|
||||||
|
|
||||||
else:
|
|
||||||
warnproblem('unknown compressed or binary file', fd)
|
|
||||||
|
|
||||||
elif has_extension(fp, 'java'):
|
|
||||||
if not os.path.isfile(fp):
|
|
||||||
continue
|
|
||||||
for line in file(fp):
|
|
||||||
if 'DexClassLoader' in line:
|
|
||||||
count += handleproblem('DexClassLoader', fd, fp)
|
|
||||||
break
|
|
||||||
|
|
||||||
elif has_extension(fp, 'gradle'):
|
|
||||||
if not os.path.isfile(fp):
|
|
||||||
continue
|
|
||||||
for i, line in enumerate(file(fp)):
|
|
||||||
i = i + 1
|
|
||||||
if any(suspect.match(line) for suspect in usual_suspects):
|
|
||||||
count += handleproblem('usual suspect at line %d' % i, fd, fp)
|
|
||||||
break
|
|
||||||
|
|
||||||
for p in scanignore:
|
|
||||||
if p not in scanignore_worked:
|
|
||||||
logging.error('Unused scanignore path: %s' % p)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
for p in scandelete:
|
|
||||||
if p not in scandelete_worked:
|
|
||||||
logging.error('Unused scandelete path: %s' % p)
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
# Presence of a jni directory without buildjni=yes might
|
|
||||||
# indicate a problem (if it's not a problem, explicitly use
|
|
||||||
# buildjni=no to bypass this check)
|
|
||||||
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
|
||||||
not thisbuild['buildjni']):
|
|
||||||
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
|
|
||||||
count += 1
|
|
||||||
|
|
||||||
return count
|
|
||||||
|
|
||||||
|
|
||||||
def natural_key(s):
|
def natural_key(s):
|
||||||
return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
|
return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
|
||||||
|
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import traceback
|
import traceback
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
import logging
|
import logging
|
||||||
@ -30,6 +31,219 @@ config = None
|
|||||||
options = None
|
options = None
|
||||||
|
|
||||||
|
|
||||||
|
def init_mime_type():
|
||||||
|
'''
|
||||||
|
There are two incompatible versions of the 'magic' module, one
|
||||||
|
that comes as part of libmagic, which is what Debian includes as
|
||||||
|
python-magic, then another called python-magic that is a separate
|
||||||
|
project that wraps libmagic. The second is 'magic' on pypi, so
|
||||||
|
both need to be supported. Then on platforms where libmagic is
|
||||||
|
not easily included, e.g. OSX and Windows, fallback to the
|
||||||
|
built-in 'mimetypes' module so this will work without
|
||||||
|
libmagic. Hence this function with the following hacks:
|
||||||
|
'''
|
||||||
|
|
||||||
|
init_path = ''
|
||||||
|
method = ''
|
||||||
|
ms = None
|
||||||
|
|
||||||
|
def mime_from_file(path):
|
||||||
|
try:
|
||||||
|
return magic.from_file(path, mime=True)
|
||||||
|
except UnicodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def mime_file(path):
|
||||||
|
try:
|
||||||
|
return ms.file(path)
|
||||||
|
except UnicodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def mime_guess_type(path):
|
||||||
|
return mimetypes.guess_type(path, strict=False)
|
||||||
|
|
||||||
|
try:
|
||||||
|
import magic
|
||||||
|
try:
|
||||||
|
ms = magic.open(magic.MIME_TYPE)
|
||||||
|
ms.load()
|
||||||
|
magic.from_file(init_path, mime=True)
|
||||||
|
method = 'from_file'
|
||||||
|
except AttributeError:
|
||||||
|
ms.file(init_path)
|
||||||
|
method = 'file'
|
||||||
|
except ImportError:
|
||||||
|
import mimetypes
|
||||||
|
mimetypes.init()
|
||||||
|
method = 'guess_type'
|
||||||
|
|
||||||
|
logging.info("Using magic method " + method)
|
||||||
|
if method == 'from_file':
|
||||||
|
return mime_from_file
|
||||||
|
if method == 'file':
|
||||||
|
return mime_file
|
||||||
|
if method == 'guess_type':
|
||||||
|
return mime_guess_type
|
||||||
|
|
||||||
|
logging.critical("unknown magic method!")
|
||||||
|
|
||||||
|
|
||||||
|
# Scan the source code in the given directory (and all subdirectories)
|
||||||
|
# and return the number of fatal problems encountered
|
||||||
|
def scan_source(build_dir, root_dir, thisbuild):
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
# Common known non-free blobs (always lower case):
|
||||||
|
usual_suspects = [
|
||||||
|
re.compile(r'.*flurryagent', re.IGNORECASE),
|
||||||
|
re.compile(r'.*paypal.*mpl', re.IGNORECASE),
|
||||||
|
re.compile(r'.*google.*analytics', re.IGNORECASE),
|
||||||
|
re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
|
||||||
|
re.compile(r'.*google.*ad.*view', re.IGNORECASE),
|
||||||
|
re.compile(r'.*google.*admob', re.IGNORECASE),
|
||||||
|
re.compile(r'.*google.*play.*services', re.IGNORECASE),
|
||||||
|
re.compile(r'.*crittercism', re.IGNORECASE),
|
||||||
|
re.compile(r'.*heyzap', re.IGNORECASE),
|
||||||
|
re.compile(r'.*jpct.*ae', re.IGNORECASE),
|
||||||
|
re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
|
||||||
|
re.compile(r'.*bugsense', re.IGNORECASE),
|
||||||
|
re.compile(r'.*crashlytics', re.IGNORECASE),
|
||||||
|
re.compile(r'.*ouya.*sdk', re.IGNORECASE),
|
||||||
|
re.compile(r'.*libspen23', re.IGNORECASE),
|
||||||
|
]
|
||||||
|
|
||||||
|
scanignore = common.getpaths(build_dir, thisbuild, 'scanignore')
|
||||||
|
scandelete = common.getpaths(build_dir, thisbuild, 'scandelete')
|
||||||
|
|
||||||
|
scanignore_worked = set()
|
||||||
|
scandelete_worked = set()
|
||||||
|
|
||||||
|
def toignore(fd):
|
||||||
|
for p in scanignore:
|
||||||
|
if fd.startswith(p):
|
||||||
|
scanignore_worked.add(p)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def todelete(fd):
|
||||||
|
for p in scandelete:
|
||||||
|
if fd.startswith(p):
|
||||||
|
scandelete_worked.add(p)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def ignoreproblem(what, fd, fp):
|
||||||
|
logging.info('Ignoring %s at %s' % (what, fd))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def removeproblem(what, fd, fp):
|
||||||
|
logging.info('Removing %s at %s' % (what, fd))
|
||||||
|
os.remove(fp)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def warnproblem(what, fd):
|
||||||
|
logging.warn('Found %s at %s' % (what, fd))
|
||||||
|
|
||||||
|
def handleproblem(what, fd, fp):
|
||||||
|
if toignore(fd):
|
||||||
|
return ignoreproblem(what, fd, fp)
|
||||||
|
if todelete(fd):
|
||||||
|
return removeproblem(what, fd, fp)
|
||||||
|
logging.error('Found %s at %s' % (what, fd))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
get_mime_type = init_mime_type()
|
||||||
|
|
||||||
|
# Iterate through all files in the source code
|
||||||
|
for r, d, f in os.walk(build_dir, topdown=True):
|
||||||
|
|
||||||
|
# It's topdown, so checking the basename is enough
|
||||||
|
for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
|
||||||
|
if ignoredir in d:
|
||||||
|
d.remove(ignoredir)
|
||||||
|
|
||||||
|
for curfile in f:
|
||||||
|
|
||||||
|
# Path (relative) to the file
|
||||||
|
fp = os.path.join(r, curfile)
|
||||||
|
fd = fp[len(build_dir) + 1:]
|
||||||
|
|
||||||
|
mime = get_mime_type(fp)
|
||||||
|
|
||||||
|
if mime == 'application/x-sharedlib':
|
||||||
|
count += handleproblem('shared library', fd, fp)
|
||||||
|
|
||||||
|
elif mime == 'application/x-archive':
|
||||||
|
count += handleproblem('static library', fd, fp)
|
||||||
|
|
||||||
|
elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
|
||||||
|
count += handleproblem('binary executable', fd, fp)
|
||||||
|
|
||||||
|
elif mime == 'application/x-java-applet':
|
||||||
|
count += handleproblem('Java compiled class', fd, fp)
|
||||||
|
|
||||||
|
elif mime in (
|
||||||
|
'application/jar',
|
||||||
|
'application/zip',
|
||||||
|
'application/java-archive',
|
||||||
|
'application/octet-stream',
|
||||||
|
'binary', ):
|
||||||
|
|
||||||
|
if common.has_extension(fp, 'apk'):
|
||||||
|
removeproblem('APK file', fd, fp)
|
||||||
|
|
||||||
|
elif common.has_extension(fp, 'jar'):
|
||||||
|
|
||||||
|
if any(suspect.match(curfile) for suspect in usual_suspects):
|
||||||
|
count += handleproblem('usual supect', fd, fp)
|
||||||
|
else:
|
||||||
|
warnproblem('JAR file', fd)
|
||||||
|
|
||||||
|
elif common.has_extension(fp, 'zip'):
|
||||||
|
warnproblem('ZIP file', fd)
|
||||||
|
|
||||||
|
else:
|
||||||
|
warnproblem('unknown compressed or binary file', fd)
|
||||||
|
|
||||||
|
elif common.has_extension(fp, 'java'):
|
||||||
|
if not os.path.isfile(fp):
|
||||||
|
continue
|
||||||
|
for line in file(fp):
|
||||||
|
if 'DexClassLoader' in line:
|
||||||
|
count += handleproblem('DexClassLoader', fd, fp)
|
||||||
|
break
|
||||||
|
|
||||||
|
elif common.has_extension(fp, 'gradle'):
|
||||||
|
if not os.path.isfile(fp):
|
||||||
|
continue
|
||||||
|
for i, line in enumerate(file(fp)):
|
||||||
|
i = i + 1
|
||||||
|
if any(suspect.match(line) for suspect in usual_suspects):
|
||||||
|
count += handleproblem('usual suspect at line %d' % i, fd, fp)
|
||||||
|
break
|
||||||
|
|
||||||
|
for p in scanignore:
|
||||||
|
if p not in scanignore_worked:
|
||||||
|
logging.error('Unused scanignore path: %s' % p)
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
for p in scandelete:
|
||||||
|
if p not in scandelete_worked:
|
||||||
|
logging.error('Unused scandelete path: %s' % p)
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
# Presence of a jni directory without buildjni=yes might
|
||||||
|
# indicate a problem (if it's not a problem, explicitly use
|
||||||
|
# buildjni=no to bypass this check)
|
||||||
|
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
||||||
|
not thisbuild['buildjni']):
|
||||||
|
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
global config, options
|
global config, options
|
||||||
@ -89,7 +303,7 @@ def main():
|
|||||||
extlib_dir, False)
|
extlib_dir, False)
|
||||||
|
|
||||||
# Do the scan...
|
# Do the scan...
|
||||||
count = common.scan_source(build_dir, root_dir, thisbuild)
|
count = scan_source(build_dir, root_dir, thisbuild)
|
||||||
if count > 0:
|
if count > 0:
|
||||||
logging.warn('Scanner found %d problems in %s (%s)' % (
|
logging.warn('Scanner found %d problems in %s (%s)' % (
|
||||||
count, appid, thisbuild['vercode']))
|
count, appid, thisbuild['vercode']))
|
||||||
|
Loading…
Reference in New Issue
Block a user