kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-06-02 22:00:12 +02:00
Code Activity

Merge branch 'ipa-support' into 'master'

Browse Source 
minimal IPA support

Closes #1067

See merge request fdroid/fdroidserver!1413
This commit is contained in:
Hans-Christoph Steiner 2024-01-11 11:13:48 +00:00
commit 22d3ba81dd
6 changed files with 201 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitlab-ci.yml
View File

@ -20,7 +20,7 @@ buildserver run-tests:
image: registry.gitlab.com/fdroid/fdroidserver:buildserver
script:
- apt-get update
- apt-get install gnupg-agent python3-babel python3-clint python3-pycountry
- apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry
- ./tests/run-tests
# make sure that translations do not cause stacktraces
- cd $CI_PROJECT_DIR/locale
@ -42,7 +42,7 @@ metadata_v0:
image: registry.gitlab.com/fdroid/fdroidserver:buildserver
variables:
GIT_DEPTH: 1000
RELEASE_COMMIT_ID: a1c4f803de8d4dc92ebd6b571a493183d14a00bf # after ArchivePolicy: 0
RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse`
script:
- git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
- cd tests
@ -154,8 +154,8 @@ ubuntu_jammy_pip:
- $pip install sdkmanager
- sdkmanager 'build-tools;33.0.0'
# pycountry is only for linting config/mirrors.yml, so its not in setup.py
- $pip install pycountry
# Install extras_require.optional from setup.py
- $pip install biplist pycountry
- $pip install dist/fdroidserver-*.tar.gz
- tar xzf dist/fdroidserver-*.tar.gz

1
MANIFEST.in
View File

@ -543,6 +543,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt
include tests/build-tools/generate.sh
include tests/check-fdroid-apk
include tests/checkupdates.TestCase
include tests/com.fake.IpaApp_1000000000001.ipa
include tests/common.TestCase
include tests/config.py
include tests/config/antiFeatures.yml

116
fdroidserver/update.py
View File

@ -49,10 +49,10 @@ from binascii import hexlify
from . import _
from . import common
from . import index
from . import metadata
from .common import DEFAULT_LOCALE
from .exception import BuildException, FDroidException, VerificationException
import fdroidserver.index
from PIL import Image, PngImagePlugin
@ -524,6 +524,94 @@ def insert_obbs(repodir, apps, apks):
break
VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$')
def version_string_to_int(version):
"""
Convert sermver version designation to version code.
Approximately convert a [Major].[Minor].[Patch] version string
consisting of numeric characters (0-9) and periods to a number. The
exponents are chosen such that it still fits in the 64bit JSON/Android range.
"""
m = VERSION_STRING_RE.match(version)
if not m:
raise ValueError(f"invalid version string '{version}'")
major = int(m.group(1))
minor = int(m.group(2))
patch = int(m.group(3))
return major * 10**12 + minor * 10**6 + patch
def parse_ipa(ipa_path, file_size, sha256):
from biplist import readPlist
ipa = {
"apkName": os.path.basename(ipa_path),
"hash": sha256,
"hashType": "sha256",
"size": file_size,
}
with zipfile.ZipFile(ipa_path) as ipa_zip:
for info in ipa_zip.infolist():
if re.match("Payload/[^/]*.app/Info.plist", info.filename):
with ipa_zip.open(info) as plist_file:
plist = readPlist(plist_file)
ipa["packageName"] = plist["CFBundleIdentifier"]
# https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring
ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
ipa["versionName"] = plist["CFBundleShortVersionString"]
return ipa
def scan_repo_for_ipas(apkcache, repodir, knownapks):
"""Scan for IPA files in a given repo directory.
Parameters
----------
apkcache
cache dictionary containting cached file infos from previous runs
repodir
repo directory to scan
knownapks
list of all known files, as per metadata.read_metadata
Returns
-------
ipas
list of file infos for ipa files in ./repo folder
cachechanged
ture if new ipa files were found and added to `apkcache`
"""
cachechanged = False
ipas = []
for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')):
ipa_name = os.path.basename(ipa_path)
file_size = os.stat(ipa_path).st_size
if file_size == 0:
raise FDroidException(_('{path} is zero size!')
.format(path=ipa_path))
sha256 = common.sha256sum(ipa_path)
ipa = apkcache.get(ipa_name, {})
if ipa.get('hash') != sha256:
ipa = fdroidserver.update.parse_ipa(ipa_path, file_size, sha256)
apkcache[ipa_name] = ipa
cachechanged = True
added = knownapks.recordapk(ipa_name, ipa['packageName'])
if added:
ipa['added'] = added
ipas.append(ipa)
return ipas, cachechanged
def translate_per_build_anti_features(apps, apks):
"""Grab the anti-features list from the build metadata.
@ -1121,7 +1209,10 @@ def insert_localized_app_metadata(apps):
def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
"""Scan a repo for all files with an extension except APK/OBB.
"""Scan a repo for all files with an extension except APK/OBB/IPA.
This allows putting all kinds of files into repostories. E.g. Media Files,
Zip archives, ...
Parameters
----------
@ -1138,22 +1229,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
repo_files = []
repodir = repodir.encode()
for name in os.listdir(repodir):
# skip files based on file extensions, that are handled elsewhere
file_extension = common.get_file_extension(name)
if file_extension in ('apk', 'obb'):
if file_extension in ('apk', 'obb', 'ipa'):
continue
# skip source tarballs generated by fdroidserver
filename = os.path.join(repodir, name)
name_utf8 = name.decode()
if filename.endswith(b'_src.tar.gz'):
logging.debug(_('skipping source tarball: {path}')
.format(path=filename.decode()))
continue
# skip all other files generated by fdroidserver
if not common.is_repo_file(filename):
continue
stat = os.stat(filename)
if stat.st_size == 0:
raise FDroidException(_('{path} is zero size!')
.format(path=filename))
# load file infos from cache if not stale
shasum = common.sha256sum(filename)
usecache = False
if name_utf8 in apkcache:
@ -1166,6 +1264,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
logging.debug(_("Ignoring stale cache data for {apkfilename}")
.format(apkfilename=name_utf8))
# scan file if info wasn't in cache
if not usecache:
logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
repo_file = collections.OrderedDict()
@ -2254,6 +2353,11 @@ def main():
options.use_date_from_apk)
cachechanged = cachechanged or fcachechanged
apks += files
ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks)
cachechanged = cachechanged or icachechanged
apks += ipas
appid_has_apks = set()
appid_has_repo_files = set()
remove_apks = []
@ -2329,7 +2433,7 @@ def main():
if len(repodirs) > 1:
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
archived_apps = prepare_apps(apps, archapks, repodirs[1])
index.make(archived_apps, archapks, repodirs[1], True)
fdroidserver.index.make(archived_apps, archapks, repodirs[1], True)
repoapps = prepare_apps(apps, apks, repodirs[0])
@ -2342,13 +2446,13 @@ def main():
app_dict = dict()
app_dict[appid] = app
if os.path.isdir(repodir):
index.make(app_dict, apks, repodir, False)
fdroidserver.index.make(app_dict, apks, repodir, False)
else:
logging.info(_('Skipping index generation for {appid}').format(appid=appid))
return
# Make the index for the main repo...
index.make(repoapps, apks, repodirs[0], False)
fdroidserver.index.make(repoapps, apks, repodirs[0], False)
git_remote = config.get('binary_transparency_remote')
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):

4
setup.py
View File

@ -108,7 +108,11 @@ setup(
'sdkmanager >= 0.6.4',
'yamllint',
],
# Some requires are only needed for very limited cases:
# * biplist is only used for parsing Apple .ipa files
# * pycountry is only for linting config/mirrors.yml
extras_require={
'optional': ['biplist', 'pycountry'],
'test': ['pyjks', 'html5print'],
'docs': [
'sphinx',

BIN
tests/com.fake.IpaApp_1000000000001.ipa Normal file
View File

Binary file not shown.

82
tests/update.TestCase
View File

@ -1922,6 +1922,86 @@ class UpdateTest(unittest.TestCase):
index['repo'][CATEGORIES_CONFIG_NAME],
)
def test_parse_ipa(self):
ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa')
result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha')
self.maxDiff = None
self.assertDictEqual(result, {
'apkName': 'com.fake.IpaApp_1000000000001.ipa',
'hash': 'fake_sha',
'hashType': 'sha256',
'packageName': 'org.onionshare.OnionShare',
'size': 'fake_size',
'versionCode': 1000000000001,
'versionName': '1.0.1',
})
class TestUpdateVersionStringToInt(unittest.TestCase):
def test_version_string_to_int(self):
self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003)
self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3)
self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0)
self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021)
self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615)
def test_version_string_to_int_value_errors(self):
with self.assertRaises(ValueError):
fdroidserver.update.version_string_to_int("1.2.3a")
with self.assertRaises(ValueError):
fdroidserver.update.version_string_to_int("asdfasdf")
with self.assertRaises(ValueError):
fdroidserver.update.version_string_to_int("1.2.-3")
with self.assertRaises(ValueError):
fdroidserver.update.version_string_to_int("-1.2.-3")
with self.assertRaises(ValueError):
fdroidserver.update.version_string_to_int("0.0.0x3")
class TestScanRepoForIpas(unittest.TestCase):
def setUp(self):
self.maxDiff = None
def test_scan_repo_for_ipas_no_cache(self):
self.maxDiff = None
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir("repo")
with open('repo/abc.Def_123.ipa', 'w') as f:
f.write('abc')
with open('repo/xyz.XXX_123.ipa', 'w') as f:
f.write('xyz')
apkcache = mock.MagicMock()
# apkcache['a'] = 1
repodir = "repo"
knownapks = mock.MagicMock()
def mocked_parse(p, s, c):
# pylint: disable=unused-argument
return {
'packageName': 'abc' if 'abc' in p else 'xyz'
}
with mock.patch('fdroidserver.update.parse_ipa', mocked_parse):
ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks)
self.assertEqual(checkchanged, True)
self.assertEqual(len(ipas), 2)
package_names_in_ipas = [x['packageName'] for x in ipas]
self.assertTrue('abc' in package_names_in_ipas)
self.assertTrue('xyz' in package_names_in_ipas)
apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls]
self.assertTrue('abc' in apkcache_setter_package_name)
self.assertTrue('xyz' in apkcache_setter_package_name)
self.assertEqual(apkcache.__setitem__.call_count, 2)
knownapks.recordapk.call_count = 2
self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls)
self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
@ -1938,4 +2018,6 @@ if __name__ == "__main__":
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(UpdateTest))
newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt))
newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas))
unittest.main(failfast=False)