1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-07-08 10:10:10 +02:00

Merge branch 'duplicate-apk-processing' into 'master'

APK processing for duplicate versionCodes and renaming

See merge request !280
This commit is contained in:
Torsten Grote 2017-06-02 14:20:20 +00:00
commit 78d4603090
11 changed files with 261 additions and 44 deletions

View File

@ -8,3 +8,24 @@ test:
- pip3 install -e .
- cd tests
- ./complete-ci-tests
# Test that the parsing of the .txt format didn't change. The metadata
# field 'Author Web Site' was added after 0.7.0, so that can't be part
# of the test.
metadata_v0:
script:
- cd tests
- cp dump_internal_metadata_format.py dump.py # since this isn't in old commits
- git checkout 0.7.0 # or any old commit of your choosing
- cd ..
- sed -i "s/'Author Email',/'Author Email',\n'Author Web Site',/" fdroidserver/metadata.py
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
- cd fdroiddata
- ../tests/dump.py
- cd ..
- git reset --hard
- git checkout master
- cd fdroiddata
- ../tests/dump.py
- sed -i "/AuthorWebSite/d" metadata/dump_*/*.yaml
- diff -uw metadata/dump_*

View File

@ -125,7 +125,7 @@ __complete_update() {
opts="-c -v -q -b -i -I -e -w"
lopts="--create-metadata --verbose --quiet --buildreport
--interactive --icons --editor --wiki --pretty --clean --delete-unknown
--nosign --use-date-from-apk"
--nosign --rename-apks --use-date-from-apk"
case "${prev}" in
-e|--editor)
_filedir

View File

@ -55,6 +55,8 @@ from .asynchronousfilereader import AsynchronousFileReader
# A signature block file with a .DSA, .RSA, or .EC extension
CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
@ -1580,6 +1582,11 @@ def natural_key(s):
class KnownApks:
"""permanent store of existing APKs with the date they were added
This is currently the only way to permanently store the "updated"
date of APKs.
"""
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
@ -1613,17 +1620,17 @@ class KnownApks:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
def recordapk(self, apk, app, default_date=None):
def recordapk(self, apkName, app, default_date=None):
'''
Record an apk (if it's new, otherwise does nothing)
Returns the date it was added as a datetime instance
'''
if apk not in self.apks:
if apkName not in self.apks:
if default_date is None:
default_date = datetime.utcnow()
self.apks[apk] = (app, default_date)
self.apks[apkName] = (app, default_date)
self.changed = True
_, added = self.apks[apk]
_, added = self.apks[apkName]
return added
# Look up information - given the 'apkname', returns (app id, date added/None).

View File

@ -294,9 +294,12 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
# Get a list of the apks for this app...
apklist = []
versionCodes = []
for apk in apks:
if apk['packageName'] == appid:
apklist.append(apk)
if apk['versionCode'] not in versionCodes:
apklist.append(apk)
versionCodes.append(apk['versionCode'])
if len(apklist) == 0:
continue
@ -361,9 +364,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
# Check for duplicates - they will make the client unhappy...
for i in range(len(apklist) - 1):
if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
raise FDroidException("duplicate versions: '%s' - '%s'" % (
apklist[i]['apkName'], apklist[i + 1]['apkName']))
first = apklist[i]
second = apklist[i + 1]
if first['versionCode'] == second['versionCode'] \
and first['sig'] == second['sig']:
if first['hash'] == second['hash']:
raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
repodir, first['apkName'], second['apkName']))
else:
raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
repodir, first['apkName'], second['apkName']))
current_version_code = 0
current_version_file = None
@ -438,7 +448,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
and repodir == 'repo': # only create these
namefield = common.config['current_version_name_source']
sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
apklinkname = sanitized_name + b'.apk'
apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
if os.path.islink(apklinkname):
os.remove(apklinkname)

View File

@ -402,10 +402,6 @@ def getsig(apkpath):
if an error occurred.
"""
# verify the jar signature is correct
if not common.verify_apk_signature(apkpath):
return None
with zipfile.ZipFile(apkpath, 'r') as apk:
certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
@ -843,8 +839,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
if not usecache:
logging.debug("Processing " + name_utf8)
repo_file = collections.OrderedDict()
repo_file['name'] = os.path.splitext(name_utf8)[0]
# TODO rename apkname globally to something more generic
repo_file['name'] = name_utf8
repo_file['apkName'] = name_utf8
repo_file['hash'] = shasum
repo_file['hashType'] = 'sha256'
@ -853,14 +849,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
# the static ID is the SHA256 unless it is set in the metadata
repo_file['packageName'] = shasum
n = name_utf8.rsplit('_', maxsplit=1)
if len(n) == 2:
packageName = n[0]
versionCode = n[1].split('.')[0]
if re.match('^-?[0-9]+$', versionCode) \
and common.is_valid_package_name(n[0]):
repo_file['packageName'] = packageName
repo_file['versionCode'] = int(versionCode)
m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
if m:
repo_file['packageName'] = m.group(1)
repo_file['versionCode'] = int(m.group(2))
srcfilename = name + b'_src.tar.gz'
if os.path.exists(os.path.join(repodir, srcfilename)):
repo_file['srcname'] = srcfilename.decode('utf-8')
@ -1089,8 +1081,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
"""
if ' ' in apkfilename:
logging.critical("Spaces in filenames are not allowed.")
return True, None, False
if options.rename_apks:
newfilename = apkfilename.replace(' ', '_')
os.rename(os.path.join(repodir, apkfilename),
os.path.join(repodir, newfilename))
apkfilename = newfilename
else:
logging.critical("Spaces in filenames are not allowed.")
return True, None, False
apkfile = os.path.join(repodir, apkfilename)
shasum = sha256sum(apkfile)
@ -1108,21 +1106,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
if not usecache:
logging.debug("Processing " + apkfilename)
apk = {}
apk['apkName'] = apkfilename
apk['hash'] = shasum
apk['hashType'] = 'sha256'
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
apk['uses-permission'] = []
apk['uses-permission-sdk-23'] = []
apk['features'] = []
apk['icons_src'] = {}
apk['icons'] = {}
apk['antiFeatures'] = set()
if has_old_openssl(apkfile):
apk['antiFeatures'].add('KnownVuln')
try:
if SdkToolsPopen(['aapt', 'version'], output=False):
@ -1147,6 +1138,42 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
logging.critical("Failed to get apk signature")
return True, None, False
if options.rename_apks:
n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
std_short_name = os.path.join(repodir, n)
if apkfile != std_short_name:
if os.path.exists(std_short_name):
std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
if apkfile != std_long_name:
if os.path.exists(std_long_name):
dupdir = os.path.join('duplicates', repodir)
if not os.path.isdir(dupdir):
os.makedirs(dupdir, exist_ok=True)
dupfile = os.path.join('duplicates', std_long_name)
logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
os.rename(apkfile, dupfile)
return True, None, False
else:
os.rename(apkfile, std_long_name)
apkfile = std_long_name
else:
os.rename(apkfile, std_short_name)
apkfile = std_short_name
apkfilename = apkfile[len(repodir) + 1:]
apk['apkName'] = apkfilename
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
# verify the jar signature is correct
if not common.verify_apk_signature(apkfile):
return True, None, False
if has_old_openssl(apkfile):
apk['antiFeatures'].add('KnownVuln')
apkzip = zipfile.ZipFile(apkfile, 'r')
# if an APK has files newer than the system time, suggest updating
@ -1498,6 +1525,8 @@ def main():
help="When configured for signed indexes, create only unsigned indexes at this stage")
parser.add_argument("--use-date-from-apk", action="store_true", default=False,
help="Use date from apk instead of current time for newly added apks")
parser.add_argument("--rename-apks", action="store_true", default=False,
help="Rename APK files that do not match package.name_123.apk")
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
@ -1517,6 +1546,9 @@ def main():
resize_all_icons(repodirs)
sys.exit(0)
if options.rename_apks:
options.clean = True
# check that icons exist now, rather than fail at the end of `fdroid update`
for k in ['repo_icon', 'archive_icon']:
if k in config:

View File

@ -281,6 +281,70 @@ class CommonTest(unittest.TestCase):
key = "val"
"""))
def test_apk_name_regex(self):
good = [
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_-123456.apk',
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456_abcdef0.apk',
'urzip_-123456.apk',
'a0_0.apk',
'Z0_0.apk',
'a0_0_abcdef0.apk',
'a_a_a_a_0_abcdef0.apk',
'a_____0.apk',
'a_____123456_abcdef0.apk',
'org.fdroid.fdroid_123456.apk',
# valid, but "_99999" is part of packageName rather than versionCode
'org.fdroid.fdroid_99999_123456.apk',
# should be valid, but I can't figure out the regex since \w includes digits
# 'πÇÇπÇÇ现代汉语通用字българскиعربي1234ö_0_123bafd.apk',
]
for name in good:
m = fdroidserver.common.APK_NAME_REGEX.match(name)
self.assertIsNotNone(m)
self.assertIn(m.group(2), ('-123456', '0', '123456'))
self.assertIn(m.group(3), ('abcdef0', None))
bad = [
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456_abcdefg.apk',
'urzip-_-198274.apk',
'urzip-_0_123bafd.apk',
'no spaces allowed_123.apk',
'0_0.apk',
'0_0_abcdef0.apk',
]
for name in bad:
self.assertIsNone(fdroidserver.common.APK_NAME_REGEX.match(name))
def test_standard_file_name_regex(self):
good = [
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_-123456.mp3',
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456.mov',
'Document_-123456.pdf',
'WTF_0.MOV',
'Z0_0.ebk',
'a_a_a_a_0.txt',
'org.fdroid.fdroid.privileged.ota_123456.zip',
'πÇÇπÇÇ现代汉语通用字българскиعربي1234ö_0.jpeg',
'a_____0.PNG',
# valid, but "_99999" is part of packageName rather than versionCode
'a_____99999_123456.zip',
'org.fdroid.fdroid_99999_123456.zip',
]
for name in good:
m = fdroidserver.common.STANDARD_FILE_NAME_REGEX.match(name)
self.assertIsNotNone(m)
self.assertIn(m.group(2), ('-123456', '0', '123456'))
bad = [
'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_abcdefg.JPEG',
'urzip-_-198274.zip',
'urzip-_123bafd.pdf',
'no spaces allowed_123.foobar',
'a_____0.',
]
for name in bad:
self.assertIsNone(fdroidserver.common.STANDARD_FILE_NAME_REGEX.match(name))
if __name__ == "__main__":
parser = optparse.OptionParser()

View File

@ -12,7 +12,7 @@
<id>fake.ota.update</id>
<added>2016-03-10</added>
<lastupdated>2016-03-10</lastupdated>
<name>fake.ota.update_1234.zip</name>
<name>fake.ota.update_1234</name>
<summary>Tests whether OTA ZIP files are being include</summary>
<desc>&lt;p&gt;F-Droid can make use of system privileges or permissions to install, update and remove applications on its own. The only way to obtain those privileges is to become a system app.&lt;/p&gt;&lt;p&gt;This is where the Privileged Extension comes in - being a separate app and much smaller, it can be installed as a system app and communicate with the main app via AIDL IPC.&lt;/p&gt;&lt;p&gt;This has several advantages:&lt;/p&gt;&lt;ul&gt;&lt;li&gt; Reduced disk usage in the system partition&lt;/li&gt;&lt;li&gt; System updates don't remove F-Droid&lt;/li&gt;&lt;li&gt; The process of installing into system via root is safer&lt;/li&gt;&lt;/ul&gt;&lt;p&gt;This is packaged as an OTA (Over-The-Air) update ZIP file. It must be installed using TWRP or other Android recovery that can flash updates to the system from the /data/data/org.fdroid.fdroid folder on the /data partition. The standalone APK is called F-Droid Privileged Extension.&lt;/p&gt;</desc>
<license>Apache-2.0</license>
@ -129,7 +129,7 @@
<application id="obb.mainpatch.current">
<id>obb.mainpatch.current</id>
<added>2016-04-23</added>
<lastupdated>2016-04-23</lastupdated>
<lastupdated>2017-06-01</lastupdated>
<name>OBB Main/Patch Current</name>
<summary></summary>
<icon>obb.mainpatch.current.1619.png</icon>

View File

@ -178,6 +178,28 @@ else
echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!'
fi
#------------------------------------------------------------------------------#
echo_header 'copy git import and run `fdroid scanner` on it'
REPOROOT=`create_test_dir`
cd $REPOROOT
touch config.py
cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
mkdir metadata
echo "Auto Name:Just A Test" > metadata/org.fdroid.ci.test.app.txt
echo "Web Site:" >> metadata/org.fdroid.ci.test.app.txt
echo "Build:0.3,300" >> metadata/org.fdroid.ci.test.app.txt
echo " commit=0.3" >> metadata/org.fdroid.ci.test.app.txt
echo " subdir=app" >> metadata/org.fdroid.ci.test.app.txt
echo " gradle=yes" >> metadata/org.fdroid.ci.test.app.txt
echo "" >> metadata/org.fdroid.ci.test.app.txt
echo "Repo:https://gitlab.com/fdroid/ci-test-app.git" >> metadata/org.fdroid.ci.test.app.txt
echo "Repo Type:git" >> metadata/org.fdroid.ci.test.app.txt
mkdir build
cp -a $WORKSPACE/tests/tmp/importer build/org.fdroid.ci.test.app
ls -l build/org.fdroid.ci.test.app
$fdroid scanner org.fdroid.ci.test.app --verbose
#------------------------------------------------------------------------------#
echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"
@ -217,6 +239,42 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml
diff $WORKSPACE/tests/repo/index.xml repo/index.xml
#------------------------------------------------------------------------------#
echo_header 'rename apks with `fdroid update --rename-apks`, --nosign for speed'
REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt', 'yml']" >> config.py
echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/
test -d repo || mkdir repo
cp $WORKSPACE/tests/urzip.apk "repo/asdfiuhk urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 ö.apk"
$fdroid update --rename-apks --pretty --nosign
test -e repo/info.guardianproject.urzip_100.apk
grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
cp $WORKSPACE/tests/urzip-release.apk repo/
$fdroid update --rename-apks --pretty --nosign
test -e repo/info.guardianproject.urzip_100.apk
test -e repo/info.guardianproject.urzip_100_b4964fd.apk
grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json
! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml
cp $WORKSPACE/tests/urzip-release.apk repo/
$fdroid update --rename-apks --pretty --nosign
test -e repo/info.guardianproject.urzip_100.apk
test -e repo/info.guardianproject.urzip_100_b4964fd.apk
test -e duplicates/repo/info.guardianproject.urzip_100_b4964fd.apk
grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json
! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml
#------------------------------------------------------------------------------#
echo_header "test metadata checks"
@ -462,6 +520,29 @@ test -e repo/index-v1.jar
export ANDROID_HOME=$STORED_ANDROID_HOME
#------------------------------------------------------------------------------#
echo_header "check duplicate files are properly handled by fdroid update"
REPOROOT=`create_test_dir`
KEYSTORE=$WORKSPACE/tests/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
mkdir $REPOROOT/metadata
cp -a $WORKSPACE/tests/metadata/obb.mainpatch.current.txt $REPOROOT/metadata
echo "accepted_formats = ['txt']" >> config.py
cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/
cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619_another-release-key.apk $REPOROOT/repo/
$fdroid update --pretty
grep -F 'obb.mainpatch.current_1619.apk' repo/index.xml repo/index-v1.json
grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index-v1.json
! grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index.xml
# die if there are exact duplicates
cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/duplicate.apk
! $fdroid update
#------------------------------------------------------------------------------#
echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"

View File

@ -1,7 +1,8 @@
fake.ota.update_1234.zip 897a92a4ccff4f415f6ba275b2af16d4ecaee60a983b215bddcb9f8964e7a24c 2016-03-10
fake.ota.update_1234.zip fake.ota.update 2016-03-10
obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01
obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20
obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23
obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01
urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk info.guardianproject.urzip 2016-06-23

View File

@ -169,21 +169,21 @@ class UpdateTest(unittest.TestCase):
self.assertTrue(False, 'TypeError!')
def testBadGetsig(self):
"""getsig() should still be able to fetch the fingerprint of bad signatures"""
# config needed to use jarsigner and keytool
config = dict()
fdroidserver.common.fill_config_defaults(config)
fdroidserver.update.config = config
apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk')
sig = self.javagetsig(apkfile)
self.assertIsNone(sig, "sig should be None: " + str(sig))
pysig = fdroidserver.update.getsig(apkfile)
self.assertIsNone(pysig, "python sig should be None: " + str(sig))
sig = fdroidserver.update.getsig(apkfile)
self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722',
"python sig should be: " + str(sig))
apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk')
sig = self.javagetsig(apkfile)
self.assertIsNone(sig, "sig should be None: " + str(sig))
pysig = fdroidserver.update.getsig(apkfile)
self.assertIsNone(pysig, "python sig should be None: " + str(sig))
sig = fdroidserver.update.getsig(apkfile)
self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722',
"python sig should be: " + str(sig))
def testScanApksAndObbs(self):
os.chdir(os.path.join(localmodule, 'tests'))
@ -200,11 +200,12 @@ class UpdateTest(unittest.TestCase):
fdroidserver.update.options = type('', (), {})()
fdroidserver.update.options.clean = True
fdroidserver.update.options.delete_unknown = True
fdroidserver.update.options.rename_apks = False
apps = fdroidserver.metadata.read_metadata(xref=True)
knownapks = fdroidserver.common.KnownApks()
apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False)
self.assertEqual(len(apks), 6)
self.assertEqual(len(apks), 7)
apk = apks[0]
self.assertEqual(apk['minSdkVersion'], '4')
self.assertEqual(apk['targetSdkVersion'], '18')