kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-03 17:50:11 +02:00
Code Activity

Merge branch 'archive-policy-fix' into 'master'

Browse Source 
Archive policy overhaul

Closes #323, #292, and #166

See merge request !291
This commit is contained in:
Hans-Christoph Steiner 2017-07-03 09:07:08 +00:00
commit e01a45d014
18 changed files with 589 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab-ci.yml
View File

@ -2,6 +2,7 @@ image: registry.gitlab.com/fdroid/ci-images:server-latest
test:
script:
- apt-get -qq update && apt-get -y dist-upgrade
- mkdir -p /usr/lib/python3.4/site-packages/
# workaround https://github.com/pypa/setuptools/issues/937
- pip3 install setuptools==33.1.1

9
examples/config.py
View File

@ -71,6 +71,15 @@ archive_description = """
The repository of older versions of applications from the main demo repository.
"""
# This allows a specific kind of insecure APK to be included in the
# 'repo' section. Since April 2017, APK signatures that use MD5 are
# no longer considered valid, jarsigner and apksigner will return an
# error when verifying. `fdroid update` will move APKs with these
# disabled signatures to the archive. This option stops that
# behavior, and lets those APKs stay part of 'repo'.
#
# allow_disabled_algorithms = True
# Normally, all apps are collected into a single app repository, like on
# https://f-droid.org. For certain situations, it is better to make a repo
# that is made up of APKs only from a single app. For example, an automated

21
fdroidserver/common.py
View File

@ -85,6 +85,7 @@ default_config = {
'gradle': 'gradle',
'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
'allow_disabled_algorithms': False,
'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
@ -2041,6 +2042,26 @@ def verify_apk_signature(apk, jar=False):
return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
def verify_old_apk_signature(apk):
"""verify the signature on an archived APK, supporting deprecated algorithms
F-Droid aims to keep every single binary that it ever published. Therefore,
it needs to be able to verify APK signatures that include deprecated/removed
algorithms. For example, jarsigner treats an MD5 signature as unsigned.
jarsigner passes unsigned APKs as "verified"! So this has to turn
on -strict then check for result 4.
"""
_java_security = os.path.join(os.getcwd(), '.java.security')
with open(_java_security, 'w') as fp:
fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
'-strict', '-verify', apk]) == 4
apk_badchars = re.compile('''[/ :;'"]''')

160
fdroidserver/update.py
View File

@ -423,20 +423,35 @@ def get_cache_file():
def get_cache():
"""
"""Get the cached dict of the APK index
Gather information about all the apk files in the repo directory,
using cached data if possible.
using cached data if possible. Some of the index operations take a
long time, like calculating the SHA-256 and verifying the APK
signature.
The cache is invalidated if the metadata version is different, or
the 'allow_disabled_algorithms' config/option is different. In
those cases, there is no easy way to know what has changed from
the cache, so just rerun the whole thing.
:return: apkcache
"""
apkcachefile = get_cache_file()
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf, encoding='utf-8')
if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
or apkcache.get('allow_disabled_algorithms') != ada:
apkcache = {}
else:
apkcache = {}
apkcache["METADATA_VERSION"] = METADATA_VERSION
apkcache['allow_disabled_algorithms'] = ada
return apkcache
@ -445,7 +460,6 @@ def write_cache(apkcache):
cache_path = os.path.dirname(apkcachefile)
if not os.path.exists(cache_path):
os.makedirs(cache_path)
apkcache["METADATA_VERSION"] = METADATA_VERSION
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
@ -1082,7 +1096,8 @@ def scan_apk_androguard(apk, apkfile):
apk['features'].append(feature)
def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
allow_disabled_algorithms=False, archive_bad_sig=False):
"""Scan the apk with the given filename in the given repo directory.
This also extracts the icons.
@ -1093,6 +1108,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
:param knownapks: known apks info
:param use_date_from_apk: use date from APK (instead of current date)
for newly added APKs
:param allow_disabled_algorithms: allow APKs with valid signatures that include
disabled algorithms in the signature (e.g. MD5)
:param archive_bad_sig: move APKs with a bad signature to the archive
:returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
"""
@ -1184,12 +1202,29 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
# verify the jar signature is correct
# verify the jar signature is correct, allow deprecated
# algorithms only if the APK is in the archive.
skipapk = False
if not common.verify_apk_signature(apkfile):
if repodir == 'archive' or allow_disabled_algorithms:
if common.verify_old_apk_signature(apkfile):
apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
else:
skipapk = True
else:
skipapk = True
if skipapk:
if archive_bad_sig:
logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
move_apk_between_sections(repodir, 'archive', apk)
else:
logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
return True, None, False
if has_known_vulnerability(apkfile):
apk['antiFeatures'].add('KnownVuln')
if 'KnownVuln' not in apk['antiFeatures']:
if has_known_vulnerability(apkfile):
apk['antiFeatures'].add('KnownVuln')
apkzip = zipfile.ZipFile(apkfile, 'r')
@ -1363,10 +1398,13 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
apks = []
for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
apkfilename = apkfile[len(repodir) + 1:]
(skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
(skip, apk, cachethis) = scan_apk(apkcache, apkfilename, repodir, knownapks,
use_date_from_apk, ada, True)
if skip:
continue
apks.append(apk)
cachechanged = cachechanged or cachethis
return apks, cachechanged
@ -1421,6 +1459,15 @@ def make_categories_txt(repodir, categories):
def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
def filter_apk_list_sorted(apk_list):
res = []
for apk in apk_list:
if apk['packageName'] == appid:
res.append(apk)
# Sort the apk list by version code. First is highest/newest.
return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
for appid, app in apps.items():
if app.ArchivePolicy:
@ -1428,60 +1475,57 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
else:
keepversions = defaultkeepversions
def filter_apk_list_sorted(apk_list):
res = []
for apk in apk_list:
if apk['packageName'] == appid:
res.append(apk)
# Sort the apk list by version code. First is highest/newest.
return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
def move_file(from_dir, to_dir, filename, ignore_missing):
from_path = os.path.join(from_dir, filename)
if ignore_missing and not os.path.exists(from_path):
return
to_path = os.path.join(to_dir, filename)
shutil.move(from_path, to_path)
logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
.format(appid, len(apks), keepversions, len(archapks)))
if len(apks) > keepversions:
apklist = filter_apk_list_sorted(apks)
current_app_apks = filter_apk_list_sorted(apks)
if len(current_app_apks) > keepversions:
# Move back the ones we don't want.
for apk in apklist[keepversions:]:
logging.info("Moving " + apk['apkName'] + " to archive")
move_file(repodir, archivedir, apk['apkName'], False)
move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
for density in all_screen_densities:
repo_icon_dir = get_icon_dir(repodir, density)
archive_icon_dir = get_icon_dir(archivedir, density)
if density not in apk['icons']:
continue
move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
if 'srcname' in apk:
move_file(repodir, archivedir, apk['srcname'], False)
for apk in current_app_apks[keepversions:]:
move_apk_between_sections(repodir, archivedir, apk)
archapks.append(apk)
apks.remove(apk)
elif len(apks) < keepversions and len(archapks) > 0:
required = keepversions - len(apks)
archapklist = filter_apk_list_sorted(archapks)
# Move forward the ones we want again.
for apk in archapklist[:required]:
logging.info("Moving " + apk['apkName'] + " from archive")
move_file(archivedir, repodir, apk['apkName'], False)
move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
for density in all_screen_densities:
repo_icon_dir = get_icon_dir(repodir, density)
archive_icon_dir = get_icon_dir(archivedir, density)
if density not in apk['icons']:
continue
move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
if 'srcname' in apk:
move_file(archivedir, repodir, apk['srcname'], False)
archapks.remove(apk)
apks.append(apk)
current_app_archapks = filter_apk_list_sorted(archapks)
if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
kept = 0
# Move forward the ones we want again, except DisableAlgorithm
for apk in current_app_archapks:
if 'DisabledAlgorithm' not in apk['antiFeatures']:
move_apk_between_sections(archivedir, repodir, apk)
archapks.remove(apk)
apks.append(apk)
kept += 1
if kept == keepversions:
break
def move_apk_between_sections(from_dir, to_dir, apk):
"""move an APK from repo to archive or vice versa"""
def _move_file(from_dir, to_dir, filename, ignore_missing):
from_path = os.path.join(from_dir, filename)
if ignore_missing and not os.path.exists(from_path):
return
to_path = os.path.join(to_dir, filename)
if not os.path.exists(to_dir):
os.mkdir(to_dir)
shutil.move(from_path, to_path)
if from_dir == to_dir:
return
logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
_move_file(from_dir, to_dir, apk['apkName'], False)
_move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
for density in all_screen_densities:
from_icon_dir = get_icon_dir(from_dir, density)
to_icon_dir = get_icon_dir(to_dir, density)
if density not in apk['icons']:
continue
_move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
if 'srcname' in apk:
_move_file(from_dir, to_dir, apk['srcname'], False)
def add_apks_to_per_app_repos(repodir, apks):
@ -1544,6 +1588,8 @@ def main():
help="Use date from apk instead of current time for newly added apks")
parser.add_argument("--rename-apks", action="store_true", default=False,
help="Rename APK files that do not match package.name_123.apk")
parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
help="Include APKs that are signed with disabled algorithms like MD5")
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W

36
tests/metadata/com.politedroid.txt Normal file
View File

@ -0,0 +1,36 @@
Categories:Time
License:GPL-3.0
Web Site:
Source Code:https://github.com/miguelvps/PoliteDroid
Issue Tracker:https://github.com/miguelvps/PoliteDroid/issues
Auto Name:Polite Droid
Summary:Calendar tool
Description:
Activates silent mode during calendar events.
.
Repo Type:git
Repo:https://github.com/miguelvps/PoliteDroid.git
Build:1.2,3
commit=6a548e4b19
target=android-10
Build:1.3,4
commit=ad865b57bf3ac59580f38485608a9b1dda4fa7dc
target=android-15
Build:1.4,5
commit=456bd615f3fbe6dff06433928cf7ea20073601fb
target=android-10
Build:1.5,6
commit=v1.5
gradle=yes
Archive Policy:4 versions
Auto Update Mode:Version v%v
Update Check Mode:Tags
Current Version:1.5
Current Version Code:6

BIN
tests/org.bitbucket.tickytacky.mirrormirror_1.apk Normal file
View File

Binary file not shown.

BIN
tests/org.bitbucket.tickytacky.mirrormirror_2.apk Normal file
View File

Binary file not shown.

BIN
tests/org.bitbucket.tickytacky.mirrormirror_3.apk Normal file
View File

Binary file not shown.

BIN
tests/org.bitbucket.tickytacky.mirrormirror_4.apk Normal file
View File

Binary file not shown.

1
tests/repo/categories.txt
View File

@ -7,3 +7,4 @@ None
Phone & SMS
Security
System
Time

BIN
tests/repo/com.politedroid_3.apk Normal file
View File

Binary file not shown.

BIN
tests/repo/com.politedroid_4.apk Normal file
View File

Binary file not shown.

BIN
tests/repo/com.politedroid_5.apk Normal file
View File

Binary file not shown.

BIN
tests/repo/com.politedroid_6.apk Normal file
View File

Binary file not shown.

65
tests/repo/index.xml
View File

@ -159,6 +159,71 @@
<sig>b4964fd759edaa54e65bb476d0276880</sig>
</package>
</application>
<application id="com.politedroid">
<id>com.politedroid</id>
<added>2017-06-23</added>
<lastupdated>2017-06-23</lastupdated>
<name>Polite Droid</name>
<summary>Calendar tool</summary>
<icon>com.politedroid.6.png</icon>
<desc>&lt;p&gt;Activates silent mode during calendar events.&lt;/p&gt;</desc>
<license>GPL-3.0</license>
<categories>Time</categories>
<category>Time</category>
<web></web>
<source>https://github.com/miguelvps/PoliteDroid</source>
<tracker>https://github.com/miguelvps/PoliteDroid/issues</tracker>
<marketversion>1.5</marketversion>
<marketvercode>6</marketvercode>
<package>
<version>1.5</version>
<versioncode>6</versioncode>
<apkname>com.politedroid_6.apk</apkname>
<hash type="sha256">70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d</hash>
<size>16578</size>
<sdkver>14</sdkver>
<targetSdkVersion>21</targetSdkVersion>
<added>2017-06-23</added>
<sig>b4964fd759edaa54e65bb476d0276880</sig>
<permissions>READ_CALENDAR,RECEIVE_BOOT_COMPLETED</permissions>
</package>
<package>
<version>1.4</version>
<versioncode>5</versioncode>
<apkname>com.politedroid_5.apk</apkname>
<hash type="sha256">5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d</hash>
<size>18817</size>
<sdkver>3</sdkver>
<targetSdkVersion>10</targetSdkVersion>
<added>2017-06-23</added>
<sig>b4964fd759edaa54e65bb476d0276880</sig>
<permissions>READ_CALENDAR,RECEIVE_BOOT_COMPLETED</permissions>
</package>
<package>
<version>1.3</version>
<versioncode>4</versioncode>
<apkname>com.politedroid_4.apk</apkname>
<hash type="sha256">c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075</hash>
<size>18489</size>
<sdkver>3</sdkver>
<targetSdkVersion>3</targetSdkVersion>
<added>2017-06-23</added>
<sig>b4964fd759edaa54e65bb476d0276880</sig>
<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
</package>
<package>
<version>1.2</version>
<versioncode>3</versioncode>
<apkname>com.politedroid_3.apk</apkname>
<hash type="sha256">665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d</hash>
<size>17552</size>
<sdkver>3</sdkver>
<targetSdkVersion>3</targetSdkVersion>
<added>2017-06-23</added>
<sig>b4964fd759edaa54e65bb476d0276880</sig>
<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
</package>
</application>
<application id="info.guardianproject.urzip">
<id>info.guardianproject.urzip</id>
<added>2016-06-23</added>

257
tests/run-tests
View File

@ -141,6 +141,8 @@ $fdroid signindex --verbose
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
test -L urzip.apk
grep -F '<application id=' repo/index.xml > /dev/null
@ -237,7 +239,250 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
# we can't easily reproduce the timestamps for things, so just hardcode them
sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml
diff $WORKSPACE/tests/repo/index.xml repo/index.xml
diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml
#------------------------------------------------------------------------------#
echo_header 'test moving lots of APKs to the archive'
REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt']" >> config.py
sed -i '/allow_disabled_algorithms/d' config.py
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/*.txt metadata/
echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
echo 'Summary:good MD5 sig, which is disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
sed -i '/Archive Policy:/d' metadata/*.txt
test -d repo || mkdir repo
cp $WORKSPACE/tests/urzip.apk \
$WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
$WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
$WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \
repo/
sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 5
test `grep '<package>' repo/index.xml | wc -l` -eq 7
#------------------------------------------------------------------------------#
echo_header 'test per-app "Archive Policy"'
REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt']" >> config.py
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
test -d repo || mkdir repo
cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/
sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 0
test `grep '<package>' repo/index.xml | wc -l` -eq 4
grep -F com.politedroid_3.apk repo/index.xml
grep -F com.politedroid_4.apk repo/index.xml
grep -F com.politedroid_5.apk repo/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e repo/com.politedroid_3.apk
test -e repo/com.politedroid_4.apk
test -e repo/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
echo "enable one app in the repo"
sed -i 's,^Archive Policy:4,Archive Policy:1,' metadata/com.politedroid.txt
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 3
test `grep '<package>' repo/index.xml | wc -l` -eq 1
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk archive/index.xml
grep -F com.politedroid_5.apk archive/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
test -e archive/com.politedroid_4.apk
test -e archive/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
echo "remove all apps from the repo"
sed -i 's,^Archive Policy:1,Archive Policy:0,' metadata/com.politedroid.txt
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 4
test `grep '<package>' repo/index.xml | wc -l` -eq 0
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk archive/index.xml
grep -F com.politedroid_5.apk archive/index.xml
grep -F com.politedroid_6.apk archive/index.xml
test -e archive/com.politedroid_3.apk
test -e archive/com.politedroid_4.apk
test -e archive/com.politedroid_5.apk
test -e archive/com.politedroid_6.apk
! test -e repo/com.politedroid_6.apk
echo "move back one from archive to the repo"
sed -i 's,^Archive Policy:0,Archive Policy:1,' metadata/com.politedroid.txt
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 3
test `grep '<package>' repo/index.xml | wc -l` -eq 1
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk archive/index.xml
grep -F com.politedroid_5.apk archive/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
test -e archive/com.politedroid_4.apk
test -e archive/com.politedroid_5.apk
! test -e archive/com.politedroid_6.apk
test -e repo/com.politedroid_6.apk
#------------------------------------------------------------------------------#
echo_header 'test moving old APKs to and from the archive'
REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt']" >> config.py
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
sed -i '/Archive Policy:/d' metadata/com.politedroid.txt
test -d repo || mkdir repo
cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/
sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 1
test `grep '<package>' repo/index.xml | wc -l` -eq 3
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk repo/index.xml
grep -F com.politedroid_5.apk repo/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
test -e repo/com.politedroid_4.apk
test -e repo/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
sed -i 's,archive_older = 3,archive_older = 1,' config.py
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 3
test `grep '<package>' repo/index.xml | wc -l` -eq 1
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk archive/index.xml
grep -F com.politedroid_5.apk archive/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
test -e archive/com.politedroid_4.apk
test -e archive/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
# disabling deletes from the archive
sed -i 's/Build:1.3,4/Build:1.3,4\n disable=testing deletion/' metadata/com.politedroid.txt
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 2
test `grep '<package>' repo/index.xml | wc -l` -eq 1
grep -F com.politedroid_3.apk archive/index.xml
! grep -F com.politedroid_4.apk archive/index.xml
grep -F com.politedroid_5.apk archive/index.xml
grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
! test -e archive/com.politedroid_4.apk
test -e archive/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
# disabling deletes from the repo, and promotes one from the archive
sed -i 's/Build:1.5,6/Build:1.5,6\n disable=testing deletion/' metadata/com.politedroid.txt
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 1
test `grep '<package>' repo/index.xml | wc -l` -eq 1
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_5.apk repo/index.xml
! grep -F com.politedroid_6.apk repo/index.xml
test -e archive/com.politedroid_3.apk
test -e repo/com.politedroid_5.apk
! test -e repo/com.politedroid_6.apk
#------------------------------------------------------------------------------#
echo_header 'test allowing disabled signatures in repo and archive'
REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt']" >> config.py
echo 'allow_disabled_algorithms = True' >> config.py
sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
echo 'Summary:good MD5 sig, disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
sed -i '/Archive Policy:/d' metadata/*.txt
test -d repo || mkdir repo
cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
$WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
$WORKSPACE/tests/urzip-badsig.apk \
repo/
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 2
test `grep '<package>' repo/index.xml | wc -l` -eq 6
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk repo/index.xml
grep -F com.politedroid_5.apk repo/index.xml
grep -F com.politedroid_6.apk repo/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_2.apk repo/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_3.apk repo/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_4.apk repo/index.xml
! grep -F urzip-badsig.apk repo/index.xml
! grep -F urzip-badsig.apk archive/index.xml
test -e archive/com.politedroid_3.apk
test -e repo/com.politedroid_4.apk
test -e repo/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
test -e repo/org.bitbucket.tickytacky.mirrormirror_2.apk
test -e repo/org.bitbucket.tickytacky.mirrormirror_3.apk
test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk
test -e archive/urzip-badsig.apk
sed -i '/allow_disabled_algorithms/d' config.py
$fdroid update --pretty --nosign
test `grep '<package>' archive/index.xml | wc -l` -eq 5
test `grep '<package>' repo/index.xml | wc -l` -eq 3
grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_2.apk archive/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_3.apk archive/index.xml
grep -F org.bitbucket.tickytacky.mirrormirror_4.apk archive/index.xml
grep -F com.politedroid_3.apk archive/index.xml
grep -F com.politedroid_4.apk repo/index.xml
grep -F com.politedroid_5.apk repo/index.xml
grep -F com.politedroid_6.apk repo/index.xml
! grep -F urzip-badsig.apk repo/index.xml
! grep -F urzip-badsig.apk archive/index.xml
test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
test -e archive/org.bitbucket.tickytacky.mirrormirror_2.apk
test -e archive/org.bitbucket.tickytacky.mirrormirror_3.apk
test -e archive/org.bitbucket.tickytacky.mirrormirror_4.apk
test -e archive/com.politedroid_3.apk
test -e archive/urzip-badsig.apk
test -e repo/com.politedroid_4.apk
test -e repo/com.politedroid_5.apk
test -e repo/com.politedroid_6.apk
#------------------------------------------------------------------------------#
@ -517,6 +762,8 @@ grep -F '<application id=' repo/index.xml > /dev/null
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
export ANDROID_HOME=$STORED_ANDROID_HOME
@ -570,6 +817,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
grep -F '<application id=' repo/index.xml > /dev/null
@ -599,6 +848,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
grep -F '<application id=' repo/index.xml > /dev/null
@ -624,6 +875,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
grep -F '<application id=' repo/index.xml > /dev/null
@ -717,6 +970,8 @@ $fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -e tmp/apkcache
! test -z tmp/apkcache
grep -F '<application id=' repo/index.xml > /dev/null
# now set fake repo_keyalias

4
tests/stats/known_apks.txt
View File

@ -1,3 +1,7 @@
com.politedroid_3.apk repo/com.politedroid 2017-06-23
com.politedroid_4.apk repo/com.politedroid 2017-06-23
com.politedroid_5.apk repo/com.politedroid 2017-06-23
com.politedroid_6.apk repo/com.politedroid 2017-06-23
fake.ota.update_1234.zip fake.ota.update 2016-03-10
obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12

95
tests/update.TestCase
View File

@ -201,12 +201,21 @@ class UpdateTest(unittest.TestCase):
fdroidserver.update.options.clean = True
fdroidserver.update.options.delete_unknown = True
fdroidserver.update.options.rename_apks = False
fdroidserver.update.options.allow_disabled_algorithms = False
apps = fdroidserver.metadata.read_metadata(xref=True)
knownapks = fdroidserver.common.KnownApks()
apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False)
self.assertEqual(len(apks), 7)
self.assertEqual(len(apks), 11)
apk = apks[0]
self.assertEqual(apk['packageName'], 'com.politedroid')
self.assertEqual(apk['versionCode'], 3)
self.assertEqual(apk['minSdkVersion'], '3')
self.assertEqual(apk['targetSdkVersion'], '3')
self.assertFalse('maxSdkVersion' in apk)
apk = apks[4]
self.assertEqual(apk['packageName'], 'obb.main.oldversion')
self.assertEqual(apk['versionCode'], 1444412523)
self.assertEqual(apk['minSdkVersion'], '4')
self.assertEqual(apk['targetSdkVersion'], '18')
self.assertFalse('maxSdkVersion' in apk)
@ -242,7 +251,7 @@ class UpdateTest(unittest.TestCase):
config = dict()
fdroidserver.common.fill_config_defaults(config)
fdroidserver.update.config = config
os.chdir(os.path.dirname(__file__))
os.chdir(os.path.join(localmodule, 'tests'))
if os.path.basename(os.getcwd()) != 'tests':
raise Exception('This test must be run in the "tests/" subdir')
@ -255,6 +264,7 @@ class UpdateTest(unittest.TestCase):
fdroidserver.update.options.clean = True
fdroidserver.update.options.rename_apks = False
fdroidserver.update.options.delete_unknown = True
fdroidserver.update.options.allow_disabled_algorithms = False
for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'):
if not os.path.exists(icon_dir):
@ -282,6 +292,87 @@ class UpdateTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(apk, frompickle)
def test_scan_apk_signed_by_disabled_algorithms(self):
os.chdir(os.path.join(localmodule, 'tests'))
if os.path.basename(os.getcwd()) != 'tests':
raise Exception('This test must be run in the "tests/" subdir')
config = dict()
fdroidserver.common.fill_config_defaults(config)
fdroidserver.update.config = config
config['ndk_paths'] = dict()
config['accepted_formats'] = ['json', 'txt', 'yml']
fdroidserver.common.config = config
fdroidserver.update.config = config
fdroidserver.update.options = type('', (), {})()
fdroidserver.update.options.clean = True
fdroidserver.update.options.verbose = True
fdroidserver.update.options.rename_apks = False
fdroidserver.update.options.delete_unknown = True
fdroidserver.update.options.allow_disabled_algorithms = False
knownapks = fdroidserver.common.KnownApks()
apksourcedir = os.getcwd()
tmpdir = os.path.join(localmodule, '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix='test_scan_apk_signed_by_disabled_algorithms-', dir=tmpdir)
print('tmptestsdir', tmptestsdir)
os.chdir(tmptestsdir)
os.mkdir('repo')
os.mkdir('archive')
# setup the repo, create icons dirs, etc.
fdroidserver.update.scan_apks({}, 'repo', knownapks)
fdroidserver.update.scan_apks({}, 'archive', knownapks)
disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ]
for apkName in disabledsigs:
shutil.copy(os.path.join(apksourcedir, apkName),
os.path.join(tmptestsdir, 'repo'))
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
allow_disabled_algorithms=True,
archive_bad_sig=False)
self.assertFalse(skip)
self.assertIsNotNone(apk)
self.assertTrue(cachechanged)
self.assertFalse(os.path.exists(os.path.join('archive', apkName)))
self.assertTrue(os.path.exists(os.path.join('repo', apkName)))
# this test only works on systems with fully updated Java/jarsigner
# that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
allow_disabled_algorithms=False,
archive_bad_sig=True)
self.assertTrue(skip)
self.assertIsNone(apk)
self.assertFalse(cachechanged)
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'archive', knownapks,
allow_disabled_algorithms=False,
archive_bad_sig=False)
self.assertFalse(skip)
self.assertIsNotNone(apk)
self.assertTrue(cachechanged)
self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
for apkName in badsigs:
shutil.copy(os.path.join(apksourcedir, apkName),
os.path.join(tmptestsdir, 'repo'))
skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
allow_disabled_algorithms=False,
archive_bad_sig=False)
self.assertTrue(skip)
self.assertIsNone(apk)
self.assertFalse(cachechanged)
def test_scan_invalid_apk(self):
os.chdir(os.path.join(localmodule, 'tests'))
if os.path.basename(os.getcwd()) != 'tests':