mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-06 11:00:13 +02:00
Merge branch 'repo-update-break-down' into 'master'
Break down the update.scan_apk() method into smaller pieces See merge request !288
This commit is contained in:
commit
0be224b3e0
@ -909,6 +909,52 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||||||
return repo_files, cachechanged
|
return repo_files, cachechanged
|
||||||
|
|
||||||
|
|
||||||
|
def scan_apk(apk_file):
|
||||||
|
"""
|
||||||
|
Scans an APK file and returns dictionary with metadata of the APK.
|
||||||
|
|
||||||
|
Attention: This does *not* verify that the APK signature is correct.
|
||||||
|
|
||||||
|
:param apk_file: The (ideally absolute) path to the APK file
|
||||||
|
:raises BuildException
|
||||||
|
:return A dict containing APK metadata
|
||||||
|
"""
|
||||||
|
apk = {
|
||||||
|
'hash': sha256sum(apk_file),
|
||||||
|
'hashType': 'sha256',
|
||||||
|
'uses-permission': [],
|
||||||
|
'uses-permission-sdk-23': [],
|
||||||
|
'features': [],
|
||||||
|
'icons_src': {},
|
||||||
|
'icons': {},
|
||||||
|
'antiFeatures': set(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if SdkToolsPopen(['aapt', 'version'], output=False):
|
||||||
|
scan_apk_aapt(apk, apk_file)
|
||||||
|
else:
|
||||||
|
scan_apk_androguard(apk, apk_file)
|
||||||
|
|
||||||
|
# Get the signature
|
||||||
|
logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
|
||||||
|
apk['sig'] = getsig(apk_file)
|
||||||
|
if not apk['sig']:
|
||||||
|
raise BuildException("Failed to get apk signature")
|
||||||
|
|
||||||
|
# Get size of the APK
|
||||||
|
apk['size'] = os.path.getsize(apk_file)
|
||||||
|
|
||||||
|
if 'minSdkVersion' not in apk:
|
||||||
|
logging.warning("No SDK version information found in {0}".format(apk_file))
|
||||||
|
apk['minSdkVersion'] = 1
|
||||||
|
|
||||||
|
# Check for known vulnerabilities
|
||||||
|
if has_known_vulnerability(apk_file):
|
||||||
|
apk['antiFeatures'].add('KnownVuln')
|
||||||
|
|
||||||
|
return apk
|
||||||
|
|
||||||
|
|
||||||
def scan_apk_aapt(apk, apkfile):
|
def scan_apk_aapt(apk, apkfile):
|
||||||
p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
|
p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
|
||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
@ -1096,9 +1142,9 @@ def scan_apk_androguard(apk, apkfile):
|
|||||||
apk['features'].append(feature)
|
apk['features'].append(feature)
|
||||||
|
|
||||||
|
|
||||||
def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
||||||
allow_disabled_algorithms=False, archive_bad_sig=False):
|
allow_disabled_algorithms=False, archive_bad_sig=False):
|
||||||
"""Scan the apk with the given filename in the given repo directory.
|
"""Processes the apk with the given filename in the given repo directory.
|
||||||
|
|
||||||
This also extracts the icons.
|
This also extracts the icons.
|
||||||
|
|
||||||
@ -1125,14 +1171,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
logging.critical("Spaces in filenames are not allowed.")
|
logging.critical("Spaces in filenames are not allowed.")
|
||||||
return True, None, False
|
return True, None, False
|
||||||
|
|
||||||
|
apk = {}
|
||||||
apkfile = os.path.join(repodir, apkfilename)
|
apkfile = os.path.join(repodir, apkfilename)
|
||||||
shasum = sha256sum(apkfile)
|
|
||||||
|
|
||||||
cachechanged = False
|
cachechanged = False
|
||||||
usecache = False
|
usecache = False
|
||||||
if apkfilename in apkcache:
|
if apkfilename in apkcache:
|
||||||
apk = apkcache[apkfilename]
|
apk = apkcache[apkfilename]
|
||||||
if apk.get('hash') == shasum:
|
if apk.get('hash') == sha256sum(apkfile):
|
||||||
logging.debug("Reading " + apkfilename + " from cache")
|
logging.debug("Reading " + apkfilename + " from cache")
|
||||||
usecache = True
|
usecache = True
|
||||||
else:
|
else:
|
||||||
@ -1140,39 +1186,17 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
|
|
||||||
if not usecache:
|
if not usecache:
|
||||||
logging.debug("Processing " + apkfilename)
|
logging.debug("Processing " + apkfilename)
|
||||||
apk = {}
|
|
||||||
apk['hash'] = shasum
|
|
||||||
apk['hashType'] = 'sha256'
|
|
||||||
apk['uses-permission'] = []
|
|
||||||
apk['uses-permission-sdk-23'] = []
|
|
||||||
apk['features'] = []
|
|
||||||
apk['icons_src'] = {}
|
|
||||||
apk['icons'] = {}
|
|
||||||
apk['antiFeatures'] = set()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if SdkToolsPopen(['aapt', 'version'], output=False):
|
apk = scan_apk(apkfile)
|
||||||
scan_apk_aapt(apk, apkfile)
|
|
||||||
else:
|
|
||||||
scan_apk_androguard(apk, apkfile)
|
|
||||||
except BuildException:
|
except BuildException:
|
||||||
|
logging.warning('Skipping "%s" with invalid signature!', apkfilename)
|
||||||
return True, None, False
|
return True, None, False
|
||||||
|
|
||||||
if 'minSdkVersion' not in apk:
|
|
||||||
logging.warn("No SDK version information found in {0}".format(apkfile))
|
|
||||||
apk['minSdkVersion'] = 1
|
|
||||||
|
|
||||||
# Check for debuggable apks...
|
# Check for debuggable apks...
|
||||||
if common.isApkAndDebuggable(apkfile):
|
if common.isApkAndDebuggable(apkfile):
|
||||||
logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
|
logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
|
||||||
|
|
||||||
# Get the signature (or md5 of, to be precise)...
|
|
||||||
logging.debug('Getting signature of {0}'.format(apkfile))
|
|
||||||
apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
|
|
||||||
if not apk['sig']:
|
|
||||||
logging.critical("Failed to get apk signature")
|
|
||||||
return True, None, False
|
|
||||||
|
|
||||||
if options.rename_apks:
|
if options.rename_apks:
|
||||||
n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
|
n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
|
||||||
std_short_name = os.path.join(repodir, n)
|
std_short_name = os.path.join(repodir, n)
|
||||||
@ -1200,7 +1224,6 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
srcfilename = apkfilename[:-4] + "_src.tar.gz"
|
srcfilename = apkfilename[:-4] + "_src.tar.gz"
|
||||||
if os.path.exists(os.path.join(repodir, srcfilename)):
|
if os.path.exists(os.path.join(repodir, srcfilename)):
|
||||||
apk['srcname'] = srcfilename
|
apk['srcname'] = srcfilename
|
||||||
apk['size'] = os.path.getsize(apkfile)
|
|
||||||
|
|
||||||
# verify the jar signature is correct, allow deprecated
|
# verify the jar signature is correct, allow deprecated
|
||||||
# algorithms only if the APK is in the archive.
|
# algorithms only if the APK is in the archive.
|
||||||
@ -1222,10 +1245,6 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
|
logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
|
||||||
return True, None, False
|
return True, None, False
|
||||||
|
|
||||||
if 'KnownVuln' not in apk['antiFeatures']:
|
|
||||||
if has_known_vulnerability(apkfile):
|
|
||||||
apk['antiFeatures'].add('KnownVuln')
|
|
||||||
|
|
||||||
apkzip = zipfile.ZipFile(apkfile, 'r')
|
apkzip = zipfile.ZipFile(apkfile, 'r')
|
||||||
|
|
||||||
# if an APK has files newer than the system time, suggest updating
|
# if an APK has files newer than the system time, suggest updating
|
||||||
@ -1240,119 +1259,20 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
dt_obj = datetime(*manifest.date_time)
|
dt_obj = datetime(*manifest.date_time)
|
||||||
checkdt = dt_obj - timedelta(1)
|
checkdt = dt_obj - timedelta(1)
|
||||||
if datetime.today() < checkdt:
|
if datetime.today() < checkdt:
|
||||||
logging.warn('System clock is older than manifest in: '
|
logging.warning('System clock is older than manifest in: '
|
||||||
+ apkfilename
|
+ apkfilename
|
||||||
+ '\nSet clock to that time using:\n'
|
+ '\nSet clock to that time using:\n'
|
||||||
+ 'sudo date -s "' + str(dt_obj) + '"')
|
+ 'sudo date -s "' + str(dt_obj) + '"')
|
||||||
|
|
||||||
iconfilename = "%s.%s.png" % (
|
# extract icons from APK zip file
|
||||||
apk['packageName'],
|
iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
|
||||||
apk['versionCode'])
|
|
||||||
|
|
||||||
# Extract the icon file...
|
|
||||||
empty_densities = []
|
|
||||||
for density in screen_densities:
|
|
||||||
if density not in apk['icons_src']:
|
|
||||||
empty_densities.append(density)
|
|
||||||
continue
|
|
||||||
iconsrc = apk['icons_src'][density]
|
|
||||||
icon_dir = get_icon_dir(repodir, density)
|
|
||||||
icondest = os.path.join(icon_dir, iconfilename)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(icondest, 'wb') as f:
|
empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
|
||||||
f.write(get_icon_bytes(apkzip, iconsrc))
|
|
||||||
apk['icons'][density] = iconfilename
|
|
||||||
except (zipfile.BadZipFile, ValueError, KeyError) as e:
|
|
||||||
logging.warning("Error retrieving icon file: %s" % (icondest))
|
|
||||||
del apk['icons_src'][density]
|
|
||||||
empty_densities.append(density)
|
|
||||||
|
|
||||||
if '-1' in apk['icons_src']:
|
|
||||||
iconsrc = apk['icons_src']['-1']
|
|
||||||
iconpath = os.path.join(
|
|
||||||
get_icon_dir(repodir, '0'), iconfilename)
|
|
||||||
with open(iconpath, 'wb') as f:
|
|
||||||
f.write(get_icon_bytes(apkzip, iconsrc))
|
|
||||||
try:
|
|
||||||
im = Image.open(iconpath)
|
|
||||||
dpi = px_to_dpi(im.size[0])
|
|
||||||
for density in screen_densities:
|
|
||||||
if density in apk['icons']:
|
|
||||||
break
|
|
||||||
if density == screen_densities[-1] or dpi >= int(density):
|
|
||||||
apk['icons'][density] = iconfilename
|
|
||||||
shutil.move(iconpath,
|
|
||||||
os.path.join(get_icon_dir(repodir, density), iconfilename))
|
|
||||||
empty_densities.remove(density)
|
|
||||||
break
|
|
||||||
except Exception as e:
|
|
||||||
logging.warn("Failed reading {0} - {1}".format(iconpath, e))
|
|
||||||
|
|
||||||
if apk['icons']:
|
|
||||||
apk['icon'] = iconfilename
|
|
||||||
|
|
||||||
apkzip.close()
|
|
||||||
|
|
||||||
# First try resizing down to not lose quality
|
|
||||||
last_density = None
|
|
||||||
for density in screen_densities:
|
|
||||||
if density not in empty_densities:
|
|
||||||
last_density = density
|
|
||||||
continue
|
|
||||||
if last_density is None:
|
|
||||||
continue
|
|
||||||
logging.debug("Density %s not available, resizing down from %s"
|
|
||||||
% (density, last_density))
|
|
||||||
|
|
||||||
last_iconpath = os.path.join(
|
|
||||||
get_icon_dir(repodir, last_density), iconfilename)
|
|
||||||
iconpath = os.path.join(
|
|
||||||
get_icon_dir(repodir, density), iconfilename)
|
|
||||||
fp = None
|
|
||||||
try:
|
|
||||||
fp = open(last_iconpath, 'rb')
|
|
||||||
im = Image.open(fp)
|
|
||||||
|
|
||||||
size = dpi_to_px(density)
|
|
||||||
|
|
||||||
im.thumbnail((size, size), Image.ANTIALIAS)
|
|
||||||
im.save(iconpath, "PNG")
|
|
||||||
empty_densities.remove(density)
|
|
||||||
except Exception as e:
|
|
||||||
logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
|
|
||||||
finally:
|
finally:
|
||||||
if fp:
|
apkzip.close() # ensure that APK zip file gets closed
|
||||||
fp.close()
|
|
||||||
|
|
||||||
# Then just copy from the highest resolution available
|
# resize existing icons for densities missing in the APK
|
||||||
last_density = None
|
fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
|
||||||
for density in reversed(screen_densities):
|
|
||||||
if density not in empty_densities:
|
|
||||||
last_density = density
|
|
||||||
continue
|
|
||||||
if last_density is None:
|
|
||||||
continue
|
|
||||||
logging.debug("Density %s not available, copying from lower density %s"
|
|
||||||
% (density, last_density))
|
|
||||||
|
|
||||||
shutil.copyfile(
|
|
||||||
os.path.join(get_icon_dir(repodir, last_density), iconfilename),
|
|
||||||
os.path.join(get_icon_dir(repodir, density), iconfilename))
|
|
||||||
|
|
||||||
empty_densities.remove(density)
|
|
||||||
|
|
||||||
for density in screen_densities:
|
|
||||||
icon_dir = get_icon_dir(repodir, density)
|
|
||||||
icondest = os.path.join(icon_dir, iconfilename)
|
|
||||||
resize_icon(icondest, density)
|
|
||||||
|
|
||||||
# Copy from icons-mdpi to icons since mdpi is the baseline density
|
|
||||||
baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
|
|
||||||
if os.path.isfile(baseline):
|
|
||||||
apk['icons']['0'] = iconfilename
|
|
||||||
shutil.copyfile(baseline,
|
|
||||||
os.path.join(get_icon_dir(repodir, '0'), iconfilename))
|
|
||||||
|
|
||||||
if use_date_from_apk and manifest.date_time[1] != 0:
|
if use_date_from_apk and manifest.date_time[1] != 0:
|
||||||
default_date_param = datetime(*manifest.date_time)
|
default_date_param = datetime(*manifest.date_time)
|
||||||
@ -1371,8 +1291,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
|
|||||||
return False, apk, cachechanged
|
return False, apk, cachechanged
|
||||||
|
|
||||||
|
|
||||||
def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
|
def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
|
||||||
"""Scan the apks in the given repo directory.
|
"""Processes the apks in the given repo directory.
|
||||||
|
|
||||||
This also extracts the icons.
|
This also extracts the icons.
|
||||||
|
|
||||||
@ -1399,7 +1319,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
|
|||||||
for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
|
for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
|
||||||
apkfilename = apkfile[len(repodir) + 1:]
|
apkfilename = apkfile[len(repodir) + 1:]
|
||||||
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
|
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
|
||||||
(skip, apk, cachethis) = scan_apk(apkcache, apkfilename, repodir, knownapks,
|
(skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
|
||||||
use_date_from_apk, ada, True)
|
use_date_from_apk, ada, True)
|
||||||
if skip:
|
if skip:
|
||||||
continue
|
continue
|
||||||
@ -1409,6 +1329,129 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
|
|||||||
return apks, cachechanged
|
return apks, cachechanged
|
||||||
|
|
||||||
|
|
||||||
|
def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir):
|
||||||
|
"""
|
||||||
|
Extracts icons from the given APK zip in various densities,
|
||||||
|
saves them into given repo directory
|
||||||
|
and stores their names in the APK metadata dictionary.
|
||||||
|
|
||||||
|
:param icon_filename: A string representing the icon's file name
|
||||||
|
:param apk: A populated dictionary containing APK metadata.
|
||||||
|
Needs to have 'icons_src' key
|
||||||
|
:param apk_zip: An opened zipfile.ZipFile of the APK file
|
||||||
|
:param repo_dir: The directory of the APK's repository
|
||||||
|
:return: A list of icon densities that are missing
|
||||||
|
"""
|
||||||
|
empty_densities = []
|
||||||
|
for density in screen_densities:
|
||||||
|
if density not in apk['icons_src']:
|
||||||
|
empty_densities.append(density)
|
||||||
|
continue
|
||||||
|
icon_src = apk['icons_src'][density]
|
||||||
|
icon_dir = get_icon_dir(repo_dir, density)
|
||||||
|
icon_dest = os.path.join(icon_dir, icon_filename)
|
||||||
|
|
||||||
|
# Extract the icon files per density
|
||||||
|
try:
|
||||||
|
with open(icon_dest, 'wb') as f:
|
||||||
|
f.write(get_icon_bytes(apk_zip, icon_src))
|
||||||
|
apk['icons'][density] = icon_filename
|
||||||
|
except (zipfile.BadZipFile, ValueError, KeyError) as e:
|
||||||
|
logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
|
||||||
|
del apk['icons_src'][density]
|
||||||
|
empty_densities.append(density)
|
||||||
|
|
||||||
|
if '-1' in apk['icons_src']:
|
||||||
|
icon_src = apk['icons_src']['-1']
|
||||||
|
icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
|
||||||
|
with open(icon_path, 'wb') as f:
|
||||||
|
f.write(get_icon_bytes(apk_zip, icon_src))
|
||||||
|
try:
|
||||||
|
im = Image.open(icon_path)
|
||||||
|
dpi = px_to_dpi(im.size[0])
|
||||||
|
for density in screen_densities:
|
||||||
|
if density in apk['icons']:
|
||||||
|
break
|
||||||
|
if density == screen_densities[-1] or dpi >= int(density):
|
||||||
|
apk['icons'][density] = icon_filename
|
||||||
|
shutil.move(icon_path,
|
||||||
|
os.path.join(get_icon_dir(repo_dir, density), icon_filename))
|
||||||
|
empty_densities.remove(density)
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
logging.warning("Failed reading {0} - {1}".format(icon_path, e))
|
||||||
|
|
||||||
|
if apk['icons']:
|
||||||
|
apk['icon'] = icon_filename
|
||||||
|
|
||||||
|
return empty_densities
|
||||||
|
|
||||||
|
|
||||||
|
def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
|
||||||
|
"""
|
||||||
|
Resize existing icons for densities missing in the APK to ensure all densities are available
|
||||||
|
|
||||||
|
:param empty_densities: A list of icon densities that are missing
|
||||||
|
:param icon_filename: A string representing the icon's file name
|
||||||
|
:param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
|
||||||
|
:param repo_dir: The directory of the APK's repository
|
||||||
|
"""
|
||||||
|
# First try resizing down to not lose quality
|
||||||
|
last_density = None
|
||||||
|
for density in screen_densities:
|
||||||
|
if density not in empty_densities:
|
||||||
|
last_density = density
|
||||||
|
continue
|
||||||
|
if last_density is None:
|
||||||
|
continue
|
||||||
|
logging.debug("Density %s not available, resizing down from %s", density, last_density)
|
||||||
|
|
||||||
|
last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
|
||||||
|
icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
|
||||||
|
fp = None
|
||||||
|
try:
|
||||||
|
fp = open(last_icon_path, 'rb')
|
||||||
|
im = Image.open(fp)
|
||||||
|
|
||||||
|
size = dpi_to_px(density)
|
||||||
|
|
||||||
|
im.thumbnail((size, size), Image.ANTIALIAS)
|
||||||
|
im.save(icon_path, "PNG")
|
||||||
|
empty_densities.remove(density)
|
||||||
|
except Exception as e:
|
||||||
|
logging.warning("Invalid image file at %s: %s", last_icon_path, e)
|
||||||
|
finally:
|
||||||
|
if fp:
|
||||||
|
fp.close()
|
||||||
|
|
||||||
|
# Then just copy from the highest resolution available
|
||||||
|
last_density = None
|
||||||
|
for density in reversed(screen_densities):
|
||||||
|
if density not in empty_densities:
|
||||||
|
last_density = density
|
||||||
|
continue
|
||||||
|
|
||||||
|
if last_density is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
shutil.copyfile(
|
||||||
|
os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
|
||||||
|
os.path.join(get_icon_dir(repo_dir, density), icon_filename)
|
||||||
|
)
|
||||||
|
empty_densities.remove(density)
|
||||||
|
|
||||||
|
for density in screen_densities:
|
||||||
|
icon_dir = get_icon_dir(repo_dir, density)
|
||||||
|
icon_dest = os.path.join(icon_dir, icon_filename)
|
||||||
|
resize_icon(icon_dest, density)
|
||||||
|
|
||||||
|
# Copy from icons-mdpi to icons since mdpi is the baseline density
|
||||||
|
baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
|
||||||
|
if os.path.isfile(baseline):
|
||||||
|
apk['icons']['0'] = icon_filename
|
||||||
|
shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
|
||||||
|
|
||||||
|
|
||||||
def apply_info_from_latest_apk(apps, apks):
|
def apply_info_from_latest_apk(apps, apks):
|
||||||
"""
|
"""
|
||||||
Some information from the apks needs to be applied up to the application level.
|
Some information from the apks needs to be applied up to the application level.
|
||||||
@ -1663,7 +1706,7 @@ def main():
|
|||||||
delete_disabled_builds(apps, apkcache, repodirs)
|
delete_disabled_builds(apps, apkcache, repodirs)
|
||||||
|
|
||||||
# Scan all apks in the main repo
|
# Scan all apks in the main repo
|
||||||
apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
|
apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
|
||||||
|
|
||||||
files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
|
files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
|
||||||
options.use_date_from_apk)
|
options.use_date_from_apk)
|
||||||
@ -1714,7 +1757,7 @@ def main():
|
|||||||
|
|
||||||
# Scan the archive repo for apks as well
|
# Scan the archive repo for apks as well
|
||||||
if len(repodirs) > 1:
|
if len(repodirs) > 1:
|
||||||
archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
|
archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
|
||||||
if cc:
|
if cc:
|
||||||
cachechanged = True
|
cachechanged = True
|
||||||
else:
|
else:
|
||||||
|
@ -21,6 +21,7 @@ if localmodule not in sys.path:
|
|||||||
sys.path.insert(0, localmodule)
|
sys.path.insert(0, localmodule)
|
||||||
|
|
||||||
import fdroidserver.common
|
import fdroidserver.common
|
||||||
|
import fdroidserver.exception
|
||||||
import fdroidserver.metadata
|
import fdroidserver.metadata
|
||||||
import fdroidserver.update
|
import fdroidserver.update
|
||||||
from fdroidserver.common import FDroidPopen
|
from fdroidserver.common import FDroidPopen
|
||||||
@ -205,7 +206,7 @@ class UpdateTest(unittest.TestCase):
|
|||||||
|
|
||||||
apps = fdroidserver.metadata.read_metadata(xref=True)
|
apps = fdroidserver.metadata.read_metadata(xref=True)
|
||||||
knownapks = fdroidserver.common.KnownApks()
|
knownapks = fdroidserver.common.KnownApks()
|
||||||
apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False)
|
apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
|
||||||
self.assertEqual(len(apks), 11)
|
self.assertEqual(len(apks), 11)
|
||||||
apk = apks[0]
|
apk = apks[0]
|
||||||
self.assertEqual(apk['packageName'], 'com.politedroid')
|
self.assertEqual(apk['packageName'], 'com.politedroid')
|
||||||
@ -242,7 +243,48 @@ class UpdateTest(unittest.TestCase):
|
|||||||
self.assertIsNone(apk.get('obbMainFile'))
|
self.assertIsNone(apk.get('obbMainFile'))
|
||||||
self.assertIsNone(apk.get('obbPatchFile'))
|
self.assertIsNone(apk.get('obbPatchFile'))
|
||||||
|
|
||||||
def testScanApkMetadata(self):
|
def test_scan_apk(self):
|
||||||
|
config = dict()
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.update.config = config
|
||||||
|
os.chdir(os.path.join(localmodule, 'tests'))
|
||||||
|
if os.path.basename(os.getcwd()) != 'tests':
|
||||||
|
raise Exception('This test must be run in the "tests/" subdir')
|
||||||
|
|
||||||
|
apk_info = fdroidserver.update.scan_apk('org.dyndns.fules.ck_20.apk')
|
||||||
|
|
||||||
|
self.assertEqual(apk_info['icons_src'], {'240': 'res/drawable-hdpi-v4/icon_launcher.png',
|
||||||
|
'120': 'res/drawable-ldpi-v4/icon_launcher.png',
|
||||||
|
'160': 'res/drawable-mdpi-v4/icon_launcher.png',
|
||||||
|
'-1': 'res/drawable-mdpi-v4/icon_launcher.png'})
|
||||||
|
self.assertEqual(apk_info['icons'], {})
|
||||||
|
self.assertEqual(apk_info['features'], [])
|
||||||
|
self.assertEqual(apk_info['antiFeatures'], set())
|
||||||
|
self.assertEqual(apk_info['versionName'], 'v1.6pre2')
|
||||||
|
self.assertEqual(apk_info['hash'],
|
||||||
|
'897486e1f857c6c0ee32ccbad0e1b8cd82f6d0e65a44a23f13f852d2b63a18c8')
|
||||||
|
self.assertEqual(apk_info['packageName'], 'org.dyndns.fules.ck')
|
||||||
|
self.assertEqual(apk_info['versionCode'], 20)
|
||||||
|
self.assertEqual(apk_info['size'], 132453)
|
||||||
|
self.assertEqual(apk_info['nativecode'],
|
||||||
|
['arm64-v8a', 'armeabi', 'armeabi-v7a', 'mips', 'mips64', 'x86', 'x86_64'])
|
||||||
|
self.assertEqual(apk_info['minSdkVersion'], '7')
|
||||||
|
self.assertEqual(apk_info['sig'], '9bf7a6a67f95688daec75eab4b1436ac')
|
||||||
|
self.assertEqual(apk_info['hashType'], 'sha256')
|
||||||
|
self.assertEqual(apk_info['targetSdkVersion'], '8')
|
||||||
|
|
||||||
|
def test_scan_apk_no_sig(self):
|
||||||
|
config = dict()
|
||||||
|
fdroidserver.common.fill_config_defaults(config)
|
||||||
|
fdroidserver.update.config = config
|
||||||
|
os.chdir(os.path.join(localmodule, 'tests'))
|
||||||
|
if os.path.basename(os.getcwd()) != 'tests':
|
||||||
|
raise Exception('This test must be run in the "tests/" subdir')
|
||||||
|
|
||||||
|
with self.assertRaises(fdroidserver.exception.BuildException):
|
||||||
|
fdroidserver.update.scan_apk('urzip-release-unsigned.apk')
|
||||||
|
|
||||||
|
def test_process_apk(self):
|
||||||
|
|
||||||
def _build_yaml_representer(dumper, data):
|
def _build_yaml_representer(dumper, data):
|
||||||
'''Creates a YAML representation of a Build instance'''
|
'''Creates a YAML representation of a Build instance'''
|
||||||
@ -274,13 +316,25 @@ class UpdateTest(unittest.TestCase):
|
|||||||
apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk']
|
apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk']
|
||||||
|
|
||||||
for apkName in apkList:
|
for apkName in apkList:
|
||||||
_, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks, False)
|
_, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks,
|
||||||
|
False)
|
||||||
# Don't care about the date added to the repo and relative apkName
|
# Don't care about the date added to the repo and relative apkName
|
||||||
del apk['added']
|
del apk['added']
|
||||||
del apk['apkName']
|
del apk['apkName']
|
||||||
# avoid AAPT application name bug
|
# avoid AAPT application name bug
|
||||||
del apk['name']
|
del apk['name']
|
||||||
|
|
||||||
|
# ensure that icons have been extracted properly
|
||||||
|
if apkName == '../urzip.apk':
|
||||||
|
self.assertEqual(apk['icon'], 'info.guardianproject.urzip.100.png')
|
||||||
|
if apkName == '../org.dyndns.fules.ck_20.apk':
|
||||||
|
self.assertEqual(apk['icon'], 'org.dyndns.fules.ck.20.png')
|
||||||
|
for density in fdroidserver.update.screen_densities:
|
||||||
|
icon_path = os.path.join(fdroidserver.update.get_icon_dir('repo', density),
|
||||||
|
apk['icon'])
|
||||||
|
self.assertTrue(os.path.isfile(icon_path))
|
||||||
|
self.assertTrue(os.path.getsize(icon_path) > 1)
|
||||||
|
|
||||||
savepath = os.path.join('metadata', 'apk', apk['packageName'] + '.yaml')
|
savepath = os.path.join('metadata', 'apk', apk['packageName'] + '.yaml')
|
||||||
# Uncomment to save APK metadata
|
# Uncomment to save APK metadata
|
||||||
# with open(savepath, 'w') as f:
|
# with open(savepath, 'w') as f:
|
||||||
@ -292,7 +346,7 @@ class UpdateTest(unittest.TestCase):
|
|||||||
self.maxDiff = None
|
self.maxDiff = None
|
||||||
self.assertEqual(apk, frompickle)
|
self.assertEqual(apk, frompickle)
|
||||||
|
|
||||||
def test_scan_apk_signed_by_disabled_algorithms(self):
|
def test_process_apk_signed_by_disabled_algorithms(self):
|
||||||
os.chdir(os.path.join(localmodule, 'tests'))
|
os.chdir(os.path.join(localmodule, 'tests'))
|
||||||
if os.path.basename(os.getcwd()) != 'tests':
|
if os.path.basename(os.getcwd()) != 'tests':
|
||||||
raise Exception('This test must be run in the "tests/" subdir')
|
raise Exception('This test must be run in the "tests/" subdir')
|
||||||
@ -318,21 +372,23 @@ class UpdateTest(unittest.TestCase):
|
|||||||
tmpdir = os.path.join(localmodule, '.testfiles')
|
tmpdir = os.path.join(localmodule, '.testfiles')
|
||||||
if not os.path.exists(tmpdir):
|
if not os.path.exists(tmpdir):
|
||||||
os.makedirs(tmpdir)
|
os.makedirs(tmpdir)
|
||||||
tmptestsdir = tempfile.mkdtemp(prefix='test_scan_apk_signed_by_disabled_algorithms-', dir=tmpdir)
|
tmptestsdir = tempfile.mkdtemp(prefix='test_process_apk_signed_by_disabled_algorithms-',
|
||||||
|
dir=tmpdir)
|
||||||
print('tmptestsdir', tmptestsdir)
|
print('tmptestsdir', tmptestsdir)
|
||||||
os.chdir(tmptestsdir)
|
os.chdir(tmptestsdir)
|
||||||
os.mkdir('repo')
|
os.mkdir('repo')
|
||||||
os.mkdir('archive')
|
os.mkdir('archive')
|
||||||
# setup the repo, create icons dirs, etc.
|
# setup the repo, create icons dirs, etc.
|
||||||
fdroidserver.update.scan_apks({}, 'repo', knownapks)
|
fdroidserver.update.process_apks({}, 'repo', knownapks)
|
||||||
fdroidserver.update.scan_apks({}, 'archive', knownapks)
|
fdroidserver.update.process_apks({}, 'archive', knownapks)
|
||||||
|
|
||||||
disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ]
|
disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ]
|
||||||
for apkName in disabledsigs:
|
for apkName in disabledsigs:
|
||||||
shutil.copy(os.path.join(apksourcedir, apkName),
|
shutil.copy(os.path.join(apksourcedir, apkName),
|
||||||
os.path.join(tmptestsdir, 'repo'))
|
os.path.join(tmptestsdir, 'repo'))
|
||||||
|
|
||||||
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
|
skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
|
||||||
|
knownapks,
|
||||||
allow_disabled_algorithms=True,
|
allow_disabled_algorithms=True,
|
||||||
archive_bad_sig=False)
|
archive_bad_sig=False)
|
||||||
self.assertFalse(skip)
|
self.assertFalse(skip)
|
||||||
@ -343,7 +399,8 @@ class UpdateTest(unittest.TestCase):
|
|||||||
|
|
||||||
# this test only works on systems with fully updated Java/jarsigner
|
# this test only works on systems with fully updated Java/jarsigner
|
||||||
# that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
|
# that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
|
||||||
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
|
skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
|
||||||
|
knownapks,
|
||||||
allow_disabled_algorithms=False,
|
allow_disabled_algorithms=False,
|
||||||
archive_bad_sig=True)
|
archive_bad_sig=True)
|
||||||
self.assertTrue(skip)
|
self.assertTrue(skip)
|
||||||
@ -352,7 +409,8 @@ class UpdateTest(unittest.TestCase):
|
|||||||
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
|
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
|
||||||
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
|
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
|
||||||
|
|
||||||
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'archive', knownapks,
|
skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'archive',
|
||||||
|
knownapks,
|
||||||
allow_disabled_algorithms=False,
|
allow_disabled_algorithms=False,
|
||||||
archive_bad_sig=False)
|
archive_bad_sig=False)
|
||||||
self.assertFalse(skip)
|
self.assertFalse(skip)
|
||||||
@ -361,19 +419,27 @@ class UpdateTest(unittest.TestCase):
|
|||||||
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
|
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
|
||||||
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
|
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
|
||||||
|
|
||||||
|
# ensure that icons have been moved to the archive as well
|
||||||
|
for density in fdroidserver.update.screen_densities:
|
||||||
|
icon_path = os.path.join(fdroidserver.update.get_icon_dir('archive', density),
|
||||||
|
apk['icon'])
|
||||||
|
self.assertTrue(os.path.isfile(icon_path))
|
||||||
|
self.assertTrue(os.path.getsize(icon_path) > 1)
|
||||||
|
|
||||||
badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
|
badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
|
||||||
for apkName in badsigs:
|
for apkName in badsigs:
|
||||||
shutil.copy(os.path.join(apksourcedir, apkName),
|
shutil.copy(os.path.join(apksourcedir, apkName),
|
||||||
os.path.join(tmptestsdir, 'repo'))
|
os.path.join(tmptestsdir, 'repo'))
|
||||||
|
|
||||||
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
|
skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
|
||||||
|
knownapks,
|
||||||
allow_disabled_algorithms=False,
|
allow_disabled_algorithms=False,
|
||||||
archive_bad_sig=False)
|
archive_bad_sig=False)
|
||||||
self.assertTrue(skip)
|
self.assertTrue(skip)
|
||||||
self.assertIsNone(apk)
|
self.assertIsNone(apk)
|
||||||
self.assertFalse(cachechanged)
|
self.assertFalse(cachechanged)
|
||||||
|
|
||||||
def test_scan_invalid_apk(self):
|
def test_process_invalid_apk(self):
|
||||||
os.chdir(os.path.join(localmodule, 'tests'))
|
os.chdir(os.path.join(localmodule, 'tests'))
|
||||||
if os.path.basename(os.getcwd()) != 'tests':
|
if os.path.basename(os.getcwd()) != 'tests':
|
||||||
raise Exception('This test must be run in the "tests/" subdir')
|
raise Exception('This test must be run in the "tests/" subdir')
|
||||||
@ -386,7 +452,8 @@ class UpdateTest(unittest.TestCase):
|
|||||||
|
|
||||||
knownapks = fdroidserver.common.KnownApks()
|
knownapks = fdroidserver.common.KnownApks()
|
||||||
apk = 'fake.ota.update_1234.zip' # this is not an APK, scanning should fail
|
apk = 'fake.ota.update_1234.zip' # this is not an APK, scanning should fail
|
||||||
(skip, apk, cachechanged) = fdroidserver.update.scan_apk({}, apk, 'repo', knownapks, False)
|
(skip, apk, cachechanged) = fdroidserver.update.process_apk({}, apk, 'repo', knownapks,
|
||||||
|
False)
|
||||||
|
|
||||||
self.assertTrue(skip)
|
self.assertTrue(skip)
|
||||||
self.assertIsNone(apk)
|
self.assertIsNone(apk)
|
||||||
|
Loading…
Reference in New Issue
Block a user