From 3f35b0b3615bd308d61859ea2834975cadaea958 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 15:14:14 +0100 Subject: [PATCH 01/28] lint: do yamllint install check once globally --- fdroidserver/lint.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index c0df0135..0fb97d9c 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -772,6 +772,11 @@ def main(): load_antiFeatures_config() load_categories_config() + if options.force_yamllint: + import yamllint # throw error if it is not installed + + yamllint # make pyflakes ignore this + # Get all apps... allapps = metadata.read_metadata(options.appid) apps = common.read_app_args(options.appid, allapps, False) @@ -791,11 +796,6 @@ def main(): if app.Disabled: continue - if options.force_yamllint: - import yamllint # throw error if it is not installed - - yamllint # make pyflakes ignore this - # only run yamllint when linting individual apps. if options.appid or options.force_yamllint: # run yamllint on app metadata From 4511da68b9ba731c34e6dfaa3d3718fa0dca540b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 16:45:12 +0100 Subject: [PATCH 02/28] lint: support linting config files --- fdroidserver/common.py | 26 +++++++++++++++++++----- fdroidserver/lint.py | 46 ++++++++++++++++++++++++++++++++++++++++-- tests/common.TestCase | 30 +++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 13aaebb0..49323357 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -361,6 +361,26 @@ def regsub_file(pattern, repl, path): f.write(text) +def config_type_check(path, data): + if Path(path).name == 'mirrors.yml': + expected_type = list + else: + expected_type = dict + if expected_type == dict: + if not isinstance(data, dict): + msg = _('{path} is not "key: value" dict, but a {datatype}!') + raise TypeError(msg.format(path=path, datatype=type(data).__name__)) + elif not isinstance(data, expected_type): + msg = _('{path} is not {expected_type}, but a {datatype}!') + raise TypeError( + msg.format( + path=path, + expected_type=expected_type.__name__, + datatype=type(data).__name__, + ) + ) + + def read_config(opts=None): """Read the repository config. @@ -401,11 +421,7 @@ def read_config(opts=None): config = yaml.safe_load(fp) if not config: config = {} - if not isinstance(config, dict): - msg = _('{path} is not "key: value" dict, but a {datatype}!') - raise TypeError( - msg.format(path=config_file, datatype=type(config).__name__) - ) + config_type_check(config_file, config) elif os.path.exists(old_config_file): logging.warning(_("""{oldfile} is deprecated, use {newfile}""") .format(oldfile=old_config_file, newfile=config_file)) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 0fb97d9c..04b03b25 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -20,6 +20,7 @@ from argparse import ArgumentParser import re import sys import platform +import ruamel.yaml import urllib.parse from pathlib import Path @@ -739,6 +740,21 @@ def check_certificate_pinned_binaries(app): return +def lint_config(arg): + path = Path(arg) + passed = True + yamllintresult = common.run_yamllint(path) + if yamllintresult: + print(yamllintresult) + passed = False + + with path.open() as fp: + data = ruamel.yaml.YAML(typ='safe').load(fp) + common.config_type_check(arg, data) + + return passed + + def main(): global config, options @@ -777,6 +793,33 @@ def main(): yamllint # make pyflakes ignore this + paths = list() + for arg in options.appid: + if ( + arg == 'config.yml' + or Path(arg).parent.name == 'config' + or Path(arg).parent.parent.name == 'config' # localized + ): + paths.append(arg) + + failed = 0 + if paths: + for path in paths: + options.appid.remove(path) + if not lint_config(path): + failed += 1 + # an empty list of appids means check all apps, avoid that if files were given + if not options.appid: + sys.exit(failed) + + if not lint_metadata(options): + failed += 1 + + if failed: + sys.exit(failed) + + +def lint_metadata(options): # Get all apps... allapps = metadata.read_metadata(options.appid) apps = common.read_app_args(options.appid, allapps, False) @@ -856,8 +899,7 @@ def main(): anywarns = True print("%s: %s" % (appid, warn)) - if anywarns: - sys.exit(1) + return not anywarns # A compiled, public domain list of official SPDX license tags. generated diff --git a/tests/common.TestCase b/tests/common.TestCase index 8a7ec438..4d9ce009 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2838,6 +2838,36 @@ class CommonTest(unittest.TestCase): with self.assertRaises(TypeError): fdroidserver.common.load_localized_config(CATEGORIES_CONFIG_NAME, 'repo') + def test_config_type_check_config_yml_dict(self): + fdroidserver.common.config_type_check('config.yml', dict()) + + def test_config_type_check_config_yml_list(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config.yml', list()) + + def test_config_type_check_config_yml_set(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config.yml', set()) + + def test_config_type_check_config_yml_str(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config.yml', str()) + + def test_config_type_check_mirrors_list(self): + fdroidserver.common.config_type_check('config/mirrors.yml', list()) + + def test_config_type_check_mirrors_dict(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config/mirrors.yml', dict()) + + def test_config_type_check_mirrors_set(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config/mirrors.yml', set()) + + def test_config_type_check_mirrors_str(self): + with self.assertRaises(TypeError): + fdroidserver.common.config_type_check('config/mirrors.yml', str()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 96fc49d7fc128aae6638216ff4c5d3d505eba7a4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 17:38:34 +0100 Subject: [PATCH 03/28] lint: check syntax of countryCode: fields for mirrors --- .gitlab-ci.yml | 6 ++- fdroidserver/lint.py | 23 +++++++++++ tests/get-country-region-data.py | 47 +++++++++++++++++++++ tests/lint.TestCase | 70 ++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100755 tests/get-country-region-data.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebdc30ce..b867e2b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,11 +15,12 @@ variables: # * python3-babel for compiling localization files # * gnupg-agent for the full signing setup # * python3-clint for fancy progress bars for users +# * python3-pycountry for linting config/mirrors.yml buildserver run-tests: image: registry.gitlab.com/fdroid/fdroidserver:buildserver script: - apt-get update - - apt-get install gnupg-agent python3-babel python3-clint + - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry - ./tests/run-tests # make sure that translations do not cause stacktraces - cd $CI_PROJECT_DIR/locale @@ -152,6 +153,9 @@ ubuntu_jammy_pip: - $pip install sdkmanager - sdkmanager 'build-tools;33.0.0' + # pycountry is only for linting config/mirrors.yml, so its not in setup.py + - $pip install pycountry + - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 04b03b25..150258ad 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from argparse import ArgumentParser +import difflib import re import sys import platform @@ -752,6 +753,28 @@ def lint_config(arg): data = ruamel.yaml.YAML(typ='safe').load(fp) common.config_type_check(arg, data) + if path.name == 'mirrors.yml': + import pycountry + + valid_country_codes = [c.alpha_2 for c in pycountry.countries] + for mirror in data: + code = mirror.get('countryCode') + if code and code not in valid_country_codes: + passed = False + msg = _( + '{path}: "{code}" is not a valid ISO_3166-1 alpha-2 country code!' + ).format(path=str(path), code=code) + if code.upper() in valid_country_codes: + m = [code.upper()] + else: + m = difflib.get_close_matches( + code.upper(), valid_country_codes, 2, 0.5 + ) + if m: + msg += ' ' + msg += _('Did you mean {code}?').format(code=', '.join(sorted(m))) + print(msg) + return passed diff --git a/tests/get-country-region-data.py b/tests/get-country-region-data.py new file mode 100755 index 00000000..f0f52e4b --- /dev/null +++ b/tests/get-country-region-data.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# This generates a list of ISO_3166-1 alpha 2 country codes for use in lint. + +import collections +import os +import re +import requests +import requests_cache +import sys +import tempfile + + +def main(): + # we want all the data + url = 'https://api.worldbank.org/v2/country?format=json&per_page=500' + r = requests.get(url, timeout=30) + data = r.json() + if data[0]['pages'] != 1: + print( + 'ERROR: %d pages in data, this script only reads one page!' + % data[0]['pages'] + ) + sys.exit(1) + + iso2Codes = set() + ISO3166_1_alpha_2_codes = set() + names = dict() + regions = collections.defaultdict(set) + for country in data[1]: + iso2Code = country['iso2Code'] + iso2Codes.add(iso2Code) + if country['region']['value'] == 'Aggregates': + continue + if re.match(r'[A-Z][A-Z]', iso2Code): + ISO3166_1_alpha_2_codes.add(iso2Code) + names[iso2Code] = country['name'] + regions[country['region']['value']].add(country['name']) + for code in sorted(ISO3166_1_alpha_2_codes): + print(f" '{code}', # " + names[code]) + + +if __name__ == "__main__": + requests_cache.install_cache( + os.path.join(tempfile.gettempdir(), os.path.basename(__file__) + '.cache') + ) + main() diff --git a/tests/lint.TestCase b/tests/lint.TestCase index d69382f0..55c314b0 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -5,6 +5,7 @@ import logging import optparse import os +import ruamel.yaml import shutil import sys import tempfile @@ -368,6 +369,75 @@ class LintTest(unittest.TestCase): app = fdroidserver.metadata.App({'Categories': ['bar']}) self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app)))) + def test_lint_config_basic_mirrors_yml(self): + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://example.com/fdroid/repo'}], fp) + self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_kenya_countryCode(self): + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'KE'}], fp) + self.assertTrue(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_invalid_countryCode(self): + """WV is "indeterminately reserved" so it should never be used.""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://foo.com/fdroid/repo', 'countryCode': 'WV'}], fp) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_alpha3_countryCode(self): + """Only ISO 3166-1 alpha 2 are supported""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump([{'url': 'https://de.com/fdroid/repo', 'countryCode': 'DEU'}], fp) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_mirrors_yml_one_invalid_countryCode(self): + """WV is "indeterminately reserved" so it should never be used.""" + os.chdir(self.testdir) + yaml = ruamel.yaml.YAML(typ='safe') + with Path('mirrors.yml').open('w') as fp: + yaml.dump( + [ + {'url': 'https://bar.com/fdroid/repo', 'countryCode': 'BA'}, + {'url': 'https://foo.com/fdroid/repo', 'countryCode': 'FO'}, + {'url': 'https://wv.com/fdroid/repo', 'countryCode': 'WV'}, + ], + fp, + ) + self.assertFalse(fdroidserver.lint.lint_config('mirrors.yml')) + + def test_lint_config_bad_mirrors_yml_dict(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('baz: [foo, bar]\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_float(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('1.0\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_int(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('1\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + + def test_lint_config_bad_mirrors_yml_str(self): + os.chdir(self.testdir) + Path('mirrors.yml').write_text('foo\n') + with self.assertRaises(TypeError): + fdroidserver.lint.lint_config('mirrors.yml') + class LintAntiFeaturesTest(unittest.TestCase): def setUp(self): From a1d9d9d885ad88285833c8b645973b9987821996 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 18:19:28 +0100 Subject: [PATCH 04/28] switch to loading mirrors.yml with ruamel to get YAML 1.2 support --- fdroidserver/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 509790bb..9a631eb8 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -26,10 +26,10 @@ import json import logging import os import re +import ruamel.yaml import shutil import tempfile import urllib.parse -import yaml import zipfile import calendar import qrcode @@ -1409,7 +1409,7 @@ def add_mirrors_to_repodict(repo_section, repodict): ) ) with mirrors_yml.open() as fp: - mirrors_config = yaml.safe_load(fp) + mirrors_config = ruamel.yaml.YAML(typ='safe').load(fp) if not isinstance(mirrors_config, list): msg = _('{path} is not list, but a {datatype}!') raise TypeError( From d7a673523d87fd409776cb2289ac0a234b31569e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 7 Dec 2023 21:36:44 +0100 Subject: [PATCH 05/28] "field will be in random order" only applies to config.py YAML only has lists, no sets or tuples, so this warning can only ever make any sense when config.py is the active config file. --- fdroidserver/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 49323357..e97cedf5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -429,12 +429,12 @@ def read_config(opts=None): code = compile(fp.read(), old_config_file, 'exec') exec(code, None, config) # nosec TODO automatically migrate - for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'): - if k in config: - if not type(config[k]) in (str, list, tuple): - logging.warning( - _("'{field}' will be in random order! Use () or [] brackets if order is important!") - .format(field=k)) + for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'): + if k in config: + if not type(config[k]) in (str, list, tuple): + logging.warning( + _("'{field}' will be in random order! Use () or [] brackets if order is important!") + .format(field=k)) # smartcardoptions must be a list since its command line args for Popen smartcardoptions = config.get('smartcardoptions') From d3abb74c888720a1d55e084308212a43c6bc6788 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 9 Jan 2024 13:15:23 +0100 Subject: [PATCH 06/28] Use git rev-parse instead of describe We only want the hash. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b867e2b7..cc32f148 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ metadata_v0: script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests - - export GITCOMMIT=`git describe` + - export GITCOMMIT=$(git rev-parse HEAD) - git checkout $RELEASE_COMMIT_ID - cd .. - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git From 0849e664e7033d291be27742decb3a78355b334e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Jan 2024 14:54:38 +0100 Subject: [PATCH 07/28] metadata_v0: use `git rev-parse` instead of `git describe` --- tests/dump_internal_metadata_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dump_internal_metadata_format.py b/tests/dump_internal_metadata_format.py index 5dade52b..24b7e911 100755 --- a/tests/dump_internal_metadata_format.py +++ b/tests/dump_internal_metadata_format.py @@ -65,7 +65,7 @@ if not os.path.isdir('metadata'): sys.exit(1) repo = git.Repo(localmodule) -savedir = os.path.join('metadata', 'dump_' + repo.git.describe()) +savedir = os.path.join('metadata', 'dump_' + repo.git.rev_parse('HEAD')) if not os.path.isdir(savedir): os.mkdir(savedir) From 69ccce06447916aa9245b68824953fcba0f7ee5c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Jan 2024 16:52:58 +0100 Subject: [PATCH 08/28] gitlab-ci: include pycountry in all master-only jobs follow up from !1418 --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc32f148..17425b8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,6 +100,7 @@ debian_testing: gnupg ipfs-cid python3-defusedxml + python3-pycountry python3-setuptools sdkmanager - python3 -c 'import fdroidserver' @@ -123,7 +124,7 @@ ubuntu_lts_ppa: - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list - apt-get update - apt-get dist-upgrade - - apt-get install --install-recommends dexdump fdroidserver git python3-setuptools sdkmanager + - apt-get install --install-recommends dexdump fdroidserver git python3-pycountry python3-setuptools sdkmanager # Test things work with a default branch other than 'master' - git config --global init.defaultBranch thisisnotmasterormain @@ -291,6 +292,7 @@ fedora_latest: python3-babel python3-matplotlib python3-pip + python3-pycountry rsync which - $pip install sdkmanager @@ -347,6 +349,9 @@ macOS: - /bin/bash --version - /bin/bash -n gradlew-fdroid tests/run-tests + # TODO remove the packages below once they are included in the Homebrew package + - $(brew --prefix fdroidserver)/libexec/bin/python3 -m pip install pycountry + # test fdroidserver from git with current package's dependencies - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests From 11d21d6b181d252a83635f7c2adfbadb9a9308a8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 11 Jan 2024 11:32:39 +0100 Subject: [PATCH 09/28] gitlab-ci: bump base commit in metadata_v0 job to get rev-parse fix !1427 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17425b8a..188a2b90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ metadata_v0: image: registry.gitlab.com/fdroid/fdroidserver:buildserver variables: GIT_DEPTH: 1000 - RELEASE_COMMIT_ID: a1c4f803de8d4dc92ebd6b571a493183d14a00bf # after ArchivePolicy: 0 + RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse` script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests From 77daf6feb686a93acaca0e6216faaa7c4cde1787 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 5 Dec 2022 14:58:08 +0100 Subject: [PATCH 10/28] Add Apple ipa support (Closes: #1067) --- fdroidserver/common.py | 2 ++ fdroidserver/update.py | 46 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e97cedf5..cfe60150 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,6 +102,7 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') +APPLE_BUNDLEiDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2015,6 +2016,7 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ + or APPLE_BUNDLEiDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a1d4dbc4..8aad6af3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -46,6 +46,7 @@ except ImportError: import collections from binascii import hexlify +from biplist import readPlist from . import _ from . import common @@ -524,6 +525,48 @@ def insert_obbs(repodir, apps, apks): break +def process_ipa(repodir, apks): + """Scan the .ipa files in a given repo directory + + Parameters + ---------- + repodir + repo directory to scan + apps + list of current, valid apps + apks + current information on all APKs + """ + def ipaWarnDelete(f, msg): + logging.warning(msg + ' ' + f) + if options.delete_unknown: + logging.error(_("Deleting unknown file: {path}").format(path=f)) + os.remove(f) + + for f in glob.glob(os.path.join(repodir, '*.ipa')): + ipa = {} + apks.append(ipa) + + ipa["apkName"] = os.path.basename(f) + ipa["hash"] = common.sha256sum(f) + ipa["hashType"] = "sha256" + ipa["size"] = os.path.getsize(f) + + with zipfile.ZipFile(f) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleversion + version = plist["CFBundleVersion"].split('.') + major = int(version.pop(0)) + minor = int(version.pop(0)) if version else 0 + patch = int(version.pop(0)) if version else 0 + ipa["packageName"] = plist["CFBundleIdentifier"] + ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch + ipa["versionName"] = plist["CFBundleShortVersionString"] + + def translate_per_build_anti_features(apps, apks): """Grab the anti-features list from the build metadata. @@ -1139,7 +1182,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repodir = repodir.encode() for name in os.listdir(repodir): file_extension = common.get_file_extension(name) - if file_extension in ('apk', 'obb'): + if file_extension in ('apk', 'obb', 'ipa'): continue filename = os.path.join(repodir, name) name_utf8 = name.decode() @@ -2128,6 +2171,7 @@ def prepare_apps(apps, apks, repodir): ------- the relevant subset of apps (as a deepcopy) """ + process_ipa(repodir, apks) apps_with_packages = get_apps_with_packages(apps, apks) apply_info_from_latest_apk(apps_with_packages, apks) insert_funding_yml_donation_links(apps) From a987341c377707a91e2703f1dc00399c56cd436b Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 9 Dec 2022 11:47:06 +0100 Subject: [PATCH 11/28] ipa: add Usage permissions --- fdroidserver/update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 8aad6af3..d8c04619 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -565,6 +565,7 @@ def process_ipa(repodir, apks): ipa["packageName"] = plist["CFBundleIdentifier"] ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} def translate_per_build_anti_features(apps, apks): From dfbb2df8397ec22f7cb5da18780a0be7a74dce72 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 9 Dec 2022 16:04:51 +0100 Subject: [PATCH 12/28] Use CFBundleShortVersionString for version code --- fdroidserver/update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index d8c04619..38b11b2c 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -557,8 +557,8 @@ def process_ipa(repodir, apks): if re.match("Payload/[^/]*.app/Info.plist", info.filename): with ipa_zip.open(info) as plist_file: plist = readPlist(plist_file) - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleversion - version = plist["CFBundleVersion"].split('.') + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + version = plist["CFBundleShortVersionString"].split('.') major = int(version.pop(0)) minor = int(version.pop(0)) if version else 0 patch = int(version.pop(0)) if version else 0 From 7d066085314bc5b37006cc44d0412bac52bd5a62 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 16:00:19 +0100 Subject: [PATCH 13/28] Move version_string_to_int into separate function --- fdroidserver/update.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 38b11b2c..30f2f5ea 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -525,8 +525,20 @@ def insert_obbs(repodir, apps, apks): break +def version_string_to_int(version): + """Approximately convert a [Major].[Minor].[Patch] version string + consisting of numeric characters (0-9) and periods to a number. The + exponents are chosen such that it still fits in the 64bit index.yaml range. + """ + version = version.split('.') + major = int(version.pop(0)) if version else 0 + minor = int(version.pop(0)) if version else 0 + patch = int(version.pop(0)) if version else 0 + return major * 10**12 + minor * 10**6 + patch + + def process_ipa(repodir, apks): - """Scan the .ipa files in a given repo directory + """Scan the .ipa files in a given repo directory. Parameters ---------- @@ -557,13 +569,9 @@ def process_ipa(repodir, apks): if re.match("Payload/[^/]*.app/Info.plist", info.filename): with ipa_zip.open(info) as plist_file: plist = readPlist(plist_file) - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - version = plist["CFBundleShortVersionString"].split('.') - major = int(version.pop(0)) - minor = int(version.pop(0)) if version else 0 - patch = int(version.pop(0)) if version else 0 ipa["packageName"] = plist["CFBundleIdentifier"] - ipa["versionCode"] = major * 10**12 + minor * 10**6 + patch + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) ipa["versionName"] = plist["CFBundleShortVersionString"] ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} From e3d319f30b8f53549c1d1449572d7f44e8bb216b Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 18:00:29 +0100 Subject: [PATCH 14/28] Update with suggestions --- fdroidserver/common.py | 4 ++-- fdroidserver/update.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index cfe60150..d52e61fd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,7 +102,7 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') -APPLE_BUNDLEiDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') +APPLE_BUNDLEIDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2016,7 +2016,7 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ - or APPLE_BUNDLEiDENTIFIER_REGEX.match(name) is not None \ + or APPLE_BUNDLEIDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 30f2f5ea..307fd5ec 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -528,7 +528,7 @@ def insert_obbs(repodir, apps, apks): def version_string_to_int(version): """Approximately convert a [Major].[Minor].[Patch] version string consisting of numeric characters (0-9) and periods to a number. The - exponents are chosen such that it still fits in the 64bit index.yaml range. + exponents are chosen such that it still fits in the 64bit JSON/Android range. """ version = version.split('.') major = int(version.pop(0)) if version else 0 From 60371093e243c0d6a6606e5a5d6168d95ee5b1d1 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 13 Jan 2023 18:11:34 +0100 Subject: [PATCH 15/28] Make python3-biplist optional --- fdroidserver/update.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 307fd5ec..9dadfd2a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -46,7 +46,6 @@ except ImportError: import collections from binascii import hexlify -from biplist import readPlist from . import _ from . import common @@ -555,25 +554,28 @@ def process_ipa(repodir, apks): logging.error(_("Deleting unknown file: {path}").format(path=f)) os.remove(f) - for f in glob.glob(os.path.join(repodir, '*.ipa')): - ipa = {} - apks.append(ipa) + ipas = glob.glob(os.path.join(repodir, '*.ipa')) + if ipas: + from biplist import readPlist + for f in ipas: + ipa = {} + apks.append(ipa) - ipa["apkName"] = os.path.basename(f) - ipa["hash"] = common.sha256sum(f) - ipa["hashType"] = "sha256" - ipa["size"] = os.path.getsize(f) + ipa["apkName"] = os.path.basename(f) + ipa["hash"] = common.sha256sum(f) + ipa["hashType"] = "sha256" + ipa["size"] = os.path.getsize(f) - with zipfile.ZipFile(f) as ipa_zip: - for info in ipa_zip.infolist(): - if re.match("Payload/[^/]*.app/Info.plist", info.filename): - with ipa_zip.open(info) as plist_file: - plist = readPlist(plist_file) - ipa["packageName"] = plist["CFBundleIdentifier"] - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) - ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + with zipfile.ZipFile(f) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + ipa["packageName"] = plist["CFBundleIdentifier"] + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) + ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} def translate_per_build_anti_features(apps, apks): From ea9374ecf633cebbe7faf6694b9d33050111ce85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 30 Nov 2023 16:27:09 +0100 Subject: [PATCH 16/28] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20=20update.py:=20f?= =?UTF-8?q?inish=20minimal=20IPA=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This add a few missing pieces to get IPA support working. (added and lastUpdated dates + caching for ipa files) --- fdroidserver/update.py | 113 ++++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9dadfd2a..0304507c 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -525,7 +525,10 @@ def insert_obbs(repodir, apps, apks): def version_string_to_int(version): - """Approximately convert a [Major].[Minor].[Patch] version string + """ + Convert sermver version designation to version code. + + Approximately convert a [Major].[Minor].[Patch] version string consisting of numeric characters (0-9) and periods to a number. The exponents are chosen such that it still fits in the 64bit JSON/Android range. """ @@ -536,46 +539,73 @@ def version_string_to_int(version): return major * 10**12 + minor * 10**6 + patch -def process_ipa(repodir, apks): - """Scan the .ipa files in a given repo directory. +def parse_ipa(ipa_path, file_size, sha256): + from biplist import readPlist + + ipa = { + "apkName": os.path.basename(ipa_path), + "hash": sha256, + "hashType": "sha256", + "size": file_size, + } + + with zipfile.ZipFile(ipa_path) as ipa_zip: + for info in ipa_zip.infolist(): + if re.match("Payload/[^/]*.app/Info.plist", info.filename): + with ipa_zip.open(info) as plist_file: + plist = readPlist(plist_file) + ipa["packageName"] = plist["CFBundleIdentifier"] + # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring + ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) + ipa["versionName"] = plist["CFBundleShortVersionString"] + ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + return ipa + + +def scan_repo_for_ipas(apkcache, repodir, knownapks): + """Scan for IPA files in a given repo directory. Parameters ---------- + apkcache + cache dictionary containting cached file infos from previous runs repodir repo directory to scan - apps - list of current, valid apps - apks - current information on all APKs + knownapks + list of all known files, as per metadata.read_metadata + + Returns + ------- + ipas + list of file infos for ipa files in ./repo folder + cachechanged + ture if new ipa files were found and added to `apkcache` """ - def ipaWarnDelete(f, msg): - logging.warning(msg + ' ' + f) - if options.delete_unknown: - logging.error(_("Deleting unknown file: {path}").format(path=f)) - os.remove(f) + cachechanged = False + ipas = [] + for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')): + ipa_name = os.path.basename(ipa_path) - ipas = glob.glob(os.path.join(repodir, '*.ipa')) - if ipas: - from biplist import readPlist - for f in ipas: - ipa = {} - apks.append(ipa) + file_size = os.stat(ipa_path).st_size + if file_size == 0: + raise FDroidException(_('{path} is zero size!') + .format(path=ipa_path)) - ipa["apkName"] = os.path.basename(f) - ipa["hash"] = common.sha256sum(f) - ipa["hashType"] = "sha256" - ipa["size"] = os.path.getsize(f) + sha256 = common.sha256sum(ipa_path) + ipa = apkcache.get(ipa_name, {}) - with zipfile.ZipFile(f) as ipa_zip: - for info in ipa_zip.infolist(): - if re.match("Payload/[^/]*.app/Info.plist", info.filename): - with ipa_zip.open(info) as plist_file: - plist = readPlist(plist_file) - ipa["packageName"] = plist["CFBundleIdentifier"] - # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring - ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) - ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} + if ipa.get('hash') != sha256: + ipa = parse_ipa(ipa_path, file_size, sha256) + apkcache[ipa_name] = ipa + cachechanged = True + + added = knownapks.recordapk(ipa_name, ipa['packageName']) + if added: + ipa['added'] = added + + ipas.append(ipa) + + return ipas, cachechanged def translate_per_build_anti_features(apps, apks): @@ -1175,7 +1205,10 @@ def insert_localized_app_metadata(apps): def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): - """Scan a repo for all files with an extension except APK/OBB. + """Scan a repo for all files with an extension except APK/OBB/IPA. + + This allows putting all kinds of files into repostories. E.g. Media Files, + Zip archives, ... Parameters ---------- @@ -1192,22 +1225,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_files = [] repodir = repodir.encode() for name in os.listdir(repodir): + # skip files based on file extensions, that are handled elsewhere file_extension = common.get_file_extension(name) if file_extension in ('apk', 'obb', 'ipa'): continue + + # skip source tarballs generated by fdroidserver filename = os.path.join(repodir, name) name_utf8 = name.decode() if filename.endswith(b'_src.tar.gz'): logging.debug(_('skipping source tarball: {path}') .format(path=filename.decode())) continue + + # skip all other files generated by fdroidserver if not common.is_repo_file(filename): continue + stat = os.stat(filename) if stat.st_size == 0: raise FDroidException(_('{path} is zero size!') .format(path=filename)) + # load file infos from cache if not stale shasum = common.sha256sum(filename) usecache = False if name_utf8 in apkcache: @@ -1220,6 +1260,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): logging.debug(_("Ignoring stale cache data for {apkfilename}") .format(apkfilename=name_utf8)) + # scan file if info wasn't in cache if not usecache: logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8)) repo_file = collections.OrderedDict() @@ -2182,7 +2223,6 @@ def prepare_apps(apps, apks, repodir): ------- the relevant subset of apps (as a deepcopy) """ - process_ipa(repodir, apks) apps_with_packages = get_apps_with_packages(apps, apks) apply_info_from_latest_apk(apps_with_packages, apks) insert_funding_yml_donation_links(apps) @@ -2309,6 +2349,11 @@ def main(): options.use_date_from_apk) cachechanged = cachechanged or fcachechanged apks += files + + ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks) + cachechanged = cachechanged or icachechanged + apks += ipas + appid_has_apks = set() appid_has_repo_files = set() remove_apks = [] From 3ee91d17775a70eae68db0bfeff53fea03a7d28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 30 Nov 2023 17:28:22 +0100 Subject: [PATCH 17/28] =?UTF-8?q?=F0=9F=A7=B4=20force=20android=20package?= =?UTF-8?q?=20names=20for=20IPAs=20for=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index d52e61fd..e97cedf5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -102,7 +102,6 @@ STRICT_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-zA-Z]+(?:\d*[a-zA-Z_]*)*)(?: VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[a-z_]+(?:\d*[a-zA-Z_]*)*)*$''', re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') -APPLE_BUNDLEIDENTIFIER_REGEX = re.compile(r'''^[a-zA-Z-.]*''') SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') @@ -2016,7 +2015,6 @@ def is_valid_package_name(name): """ return VALID_APPLICATION_ID_REGEX.match(name) is not None \ - or APPLE_BUNDLEIDENTIFIER_REGEX.match(name) is not None \ or FDROID_PACKAGE_NAME_REGEX.match(name) is not None From c288317530971f036c7170941f359040579441a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 18 Dec 2023 12:58:37 +0100 Subject: [PATCH 18/28] =?UTF-8?q?=F0=9F=AA=A8=20version=20string=20convers?= =?UTF-8?q?ion:=20error=20handling+tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 13 +++++++++---- tests/update.TestCase | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0304507c..7934ffa8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -524,6 +524,9 @@ def insert_obbs(repodir, apps, apks): break +VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$') + + def version_string_to_int(version): """ Convert sermver version designation to version code. @@ -532,10 +535,12 @@ def version_string_to_int(version): consisting of numeric characters (0-9) and periods to a number. The exponents are chosen such that it still fits in the 64bit JSON/Android range. """ - version = version.split('.') - major = int(version.pop(0)) if version else 0 - minor = int(version.pop(0)) if version else 0 - patch = int(version.pop(0)) if version else 0 + m = VERSION_STRING_RE.match(version) + if not m: + raise ValueError(f"invalid version string '{version}'") + major = int(m.group(1)) + minor = int(m.group(2)) + patch = int(m.group(3)) return major * 10**12 + minor * 10**6 + patch diff --git a/tests/update.TestCase b/tests/update.TestCase index abce3a30..0bc78462 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1923,6 +1923,28 @@ class UpdateTest(unittest.TestCase): ) +class TestUpdateVersionStringToInt(unittest.TestCase): + + def test_version_string_to_int(self): + self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3) + self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0) + self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021) + self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615) + + def test_version_string_to_int_value_errors(self): + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.3a") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("asdfasdf") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("-1.2.-3") + with self.assertRaises(ValueError): + fdroidserver.update.version_string_to_int("0.0.0x3") + + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -1938,4 +1960,5 @@ if __name__ == "__main__": newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) + newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt)) unittest.main(failfast=False) From 995118bcd20a55fda6545fb087a181b48a3d98ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 18 Dec 2023 13:35:06 +0100 Subject: [PATCH 19/28] =?UTF-8?q?=F0=9F=A5=94=20add=20strapped=20IPA=20fil?= =?UTF-8?q?e=20and=20test=20for=20parse=5Fipa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 1 - tests/com.fake.IpaApp_1000000000001.ipa | Bin 0 -> 1722 bytes tests/update.TestCase | 13 +++++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/com.fake.IpaApp_1000000000001.ipa diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 7934ffa8..b5b956a2 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -563,7 +563,6 @@ def parse_ipa(ipa_path, file_size, sha256): # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"]) ipa["versionName"] = plist["CFBundleShortVersionString"] - ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k} return ipa diff --git a/tests/com.fake.IpaApp_1000000000001.ipa b/tests/com.fake.IpaApp_1000000000001.ipa new file mode 100644 index 0000000000000000000000000000000000000000..d392cb940a8585307deb50d6f791ebc18fbd7a37 GIT binary patch literal 1722 zcma)+c{J2(9L9fRY{wq5gvl}ySuVO)lCjJ%7ulN8SQ29i2{X2i%C#gSVT|QoGKiUs zFry@K#buCPOlWLlzho&YijJF3sy}qjec$uG@Av(l_j#W8uaBJ-n3E6K9lVL@XTKC* zH(o#lu){`N3c`A6IXDUcZVzByyXzfD9I*SW5(n?668qfTQ(54!CODiX7KhW?6YzeC zLFZBDmpB0c&IB@58B};Ov)7%$t%tY z*yzN7dp0iFzNeFUrtNp4{imKYa#iXV+h?a*e3`eDb7~kRl6f()H5Q^IC zVH!&xs7#c1t-QHsy6h_P-x1nQbjg(rP;?4QvU3Tb&8rJ5AD454hlCDQfk1&U5}0WHjF!=0b;9^^<2rg=&b|cbDNFAPguv> z{13u>;}iZ?#5>B3yjvG4ZiAC7RTAtqaYv*}+tY^C+{M%Q;F4ELz2zz$d_ngLDc#Ij zC+>pq@IsOUYG|$pHLr`3V&(T`NeT1Odq1wN0?qGmg0s);xi5C)!}%Mj5h67u(TnPn z%pF7~zGwl{?-Vvk3J@}255&EHl#4v|G)PUxN`eDo$g0qCge#>Neu`8fH%%3m@V=U7QrTUBbAo7|e-Nr4f~TRcRT&O`95<{Oob}%N z6TPjKdos&UT%?HSTc?1Ha0Kk#GB^>+`1o-&hE!AdyJHLnQ$Md}q`RL*p bpZ!C=)BXR&adGed3H*6VfB+zcYxnjahOgOe literal 0 HcmV?d00001 diff --git a/tests/update.TestCase b/tests/update.TestCase index 0bc78462..9d1dea23 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1922,6 +1922,19 @@ class UpdateTest(unittest.TestCase): index['repo'][CATEGORIES_CONFIG_NAME], ) + def test_parse_ipa(self): + result = fdroidserver.update.parse_ipa('./com.fake.IpaApp_1000000000001.ipa', 'fake_size', 'fake_sha') + self.maxDiff = None + self.assertDictEqual(result, { + 'apkName': 'com.fake.IpaApp_1000000000001.ipa', + 'hash': 'fake_sha', + 'hashType': 'sha256', + 'packageName': 'org.onionshare.OnionShare', + 'size': 'fake_size', + 'versionCode': 1000000000001, + 'versionName': '1.0.1', + }) + class TestUpdateVersionStringToInt(unittest.TestCase): From 7211e9f9b43433154231ce25ef7f724ed9933fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 20 Dec 2023 04:11:05 +0100 Subject: [PATCH 20/28] =?UTF-8?q?=F0=9F=8D=B2=20add=20unit=20test=20for=20?= =?UTF-8?q?update.scan=5Frepo=5Ffor=5Fipas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/update.py | 10 ++++----- tests/update.TestCase | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b5b956a2..26e248d5 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -49,10 +49,10 @@ from binascii import hexlify from . import _ from . import common -from . import index from . import metadata from .common import DEFAULT_LOCALE from .exception import BuildException, FDroidException, VerificationException +import fdroidserver.index from PIL import Image, PngImagePlugin @@ -599,7 +599,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks): ipa = apkcache.get(ipa_name, {}) if ipa.get('hash') != sha256: - ipa = parse_ipa(ipa_path, file_size, sha256) + ipa = fdroidserver.update.parse_ipa(ipa_path, file_size, sha256) apkcache[ipa_name] = ipa cachechanged = True @@ -2433,7 +2433,7 @@ def main(): if len(repodirs) > 1: archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) archived_apps = prepare_apps(apps, archapks, repodirs[1]) - index.make(archived_apps, archapks, repodirs[1], True) + fdroidserver.index.make(archived_apps, archapks, repodirs[1], True) repoapps = prepare_apps(apps, apks, repodirs[0]) @@ -2446,13 +2446,13 @@ def main(): app_dict = dict() app_dict[appid] = app if os.path.isdir(repodir): - index.make(app_dict, apks, repodir, False) + fdroidserver.index.make(app_dict, apks, repodir, False) else: logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return # Make the index for the main repo... - index.make(repoapps, apks, repodirs[0], False) + fdroidserver.index.make(repoapps, apks, repodirs[0], False) git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): diff --git a/tests/update.TestCase b/tests/update.TestCase index 9d1dea23..d8f278b6 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1958,6 +1958,54 @@ class TestUpdateVersionStringToInt(unittest.TestCase): fdroidserver.update.version_string_to_int("0.0.0x3") +class TestScanRepoForIpas(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_scan_repo_for_ipas_no_cache(self): + self.maxDiff = None + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir("repo") + with open('repo/abc.Def_123.ipa', 'w') as f: + f.write('abc') + with open('repo/xyz.XXX_123.ipa', 'w') as f: + f.write('xyz') + + apkcache = mock.MagicMock() + # apkcache['a'] = 1 + repodir = "repo" + knownapks = mock.MagicMock() + + def mocked_parse(p, s, c): + return { + 'packageName': 'abc' if 'abc' in p else 'xyz' + } + + with mock.patch('fdroidserver.update.parse_ipa', mocked_parse): + ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks) + + self.assertEqual(checkchanged, True) + self.assertEqual(len(ipas), 2) + self.assertEqual(ipas[0]['packageName'], 'xyz') + self.assertEqual(ipas[1]['packageName'], 'abc') + + self.assertEqual(apkcache.__setitem__.mock_calls[0].args[1]['packageName'], 'xyz') + self.assertEqual(apkcache.__setitem__.mock_calls[1].args[1]['packageName'], 'abc') + self.assertEqual(apkcache.__setitem__.call_count, 2) + + knownapks.recordapk.call_count = 2 + self.assertEqual( + knownapks.recordapk.mock_calls[0], + unittest.mock.call('xyz.XXX_123.ipa', 'xyz'), + ) + # skipping one call here, because accessing `if added:` shows up in mock_calls + self.assertEqual( + knownapks.recordapk.mock_calls[2], + unittest.mock.call('abc.Def_123.ipa', 'abc'), + ) + + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -1974,4 +2022,5 @@ if __name__ == "__main__": newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) newSuite.addTest(unittest.makeSuite(TestUpdateVersionStringToInt)) + newSuite.addTest(unittest.makeSuite(TestScanRepoForIpas)) unittest.main(failfast=False) From 881943a0db430db07c4ad33f8bad73b00f5ab121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 30 Dec 2023 00:06:16 +0100 Subject: [PATCH 21/28] =?UTF-8?q?=F0=9F=A5=94=20install=20biplist=20for=20?= =?UTF-8?q?ci=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit biplist is an optional python dependency required for processing iOS apps. (.ipa files) --- .gitlab-ci.yml | 6 +++--- setup.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 188a2b90..00fd097c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ buildserver run-tests: image: registry.gitlab.com/fdroid/fdroidserver:buildserver script: - apt-get update - - apt-get install gnupg-agent python3-babel python3-clint python3-pycountry + - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry - ./tests/run-tests # make sure that translations do not cause stacktraces - cd $CI_PROJECT_DIR/locale @@ -154,8 +154,8 @@ ubuntu_jammy_pip: - $pip install sdkmanager - sdkmanager 'build-tools;33.0.0' - # pycountry is only for linting config/mirrors.yml, so its not in setup.py - - $pip install pycountry + # Install extras_require.optional from setup.py + - $pip install biplist pycountry - $pip install dist/fdroidserver-*.tar.gz - tar xzf dist/fdroidserver-*.tar.gz diff --git a/setup.py b/setup.py index 522c3377..afff96b4 100755 --- a/setup.py +++ b/setup.py @@ -108,7 +108,11 @@ setup( 'sdkmanager >= 0.6.4', 'yamllint', ], + # Some requires are only needed for very limited cases: + # * biplist is only used for parsing Apple .ipa files + # * pycountry is only for linting config/mirrors.yml extras_require={ + 'optional': ['biplist', 'pycountry'], 'test': ['pyjks', 'html5print'], 'docs': [ 'sphinx', From 8b5a61bb257fefeeadcc4d30141881294e656f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 30 Dec 2023 13:51:37 +0100 Subject: [PATCH 22/28] =?UTF-8?q?=E2=9B=B0=EF=B8=8F=20=20make=20ipa=20rela?= =?UTF-8?q?ted=20test=20cases=20more=20robust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MANIFEST.in | 1 + tests/update.TestCase | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1aed9975..05a022b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -543,6 +543,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/generate.sh include tests/check-fdroid-apk include tests/checkupdates.TestCase +include tests/com.fake.IpaApp_1000000000001.ipa include tests/common.TestCase include tests/config.py include tests/config/antiFeatures.yml diff --git a/tests/update.TestCase b/tests/update.TestCase index d8f278b6..f1c07fd9 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1923,7 +1923,8 @@ class UpdateTest(unittest.TestCase): ) def test_parse_ipa(self): - result = fdroidserver.update.parse_ipa('./com.fake.IpaApp_1000000000001.ipa', 'fake_size', 'fake_sha') + ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa') + result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha') self.maxDiff = None self.assertDictEqual(result, { 'apkName': 'com.fake.IpaApp_1000000000001.ipa', @@ -1978,6 +1979,7 @@ class TestScanRepoForIpas(unittest.TestCase): knownapks = mock.MagicMock() def mocked_parse(p, s, c): + # pylint: disable=unused-argument return { 'packageName': 'abc' if 'abc' in p else 'xyz' } @@ -1987,23 +1989,18 @@ class TestScanRepoForIpas(unittest.TestCase): self.assertEqual(checkchanged, True) self.assertEqual(len(ipas), 2) - self.assertEqual(ipas[0]['packageName'], 'xyz') - self.assertEqual(ipas[1]['packageName'], 'abc') + package_names_in_ipas = [x['packageName'] for x in ipas] + self.assertTrue('abc' in package_names_in_ipas) + self.assertTrue('xyz' in package_names_in_ipas) - self.assertEqual(apkcache.__setitem__.mock_calls[0].args[1]['packageName'], 'xyz') - self.assertEqual(apkcache.__setitem__.mock_calls[1].args[1]['packageName'], 'abc') + apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls] + self.assertTrue('abc' in apkcache_setter_package_name) + self.assertTrue('xyz' in apkcache_setter_package_name) self.assertEqual(apkcache.__setitem__.call_count, 2) knownapks.recordapk.call_count = 2 - self.assertEqual( - knownapks.recordapk.mock_calls[0], - unittest.mock.call('xyz.XXX_123.ipa', 'xyz'), - ) - # skipping one call here, because accessing `if added:` shows up in mock_calls - self.assertEqual( - knownapks.recordapk.mock_calls[2], - unittest.mock.call('abc.Def_123.ipa', 'abc'), - ) + self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls) + self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls) if __name__ == "__main__": From 6eee83db478b99892c13535054b7971dcf7b55cc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 11 Jan 2024 15:45:54 +0100 Subject: [PATCH 23/28] run black on new .ipa test cases --- tests/update.TestCase | 89 +++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/tests/update.TestCase b/tests/update.TestCase index f1c07fd9..cf8a222a 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -167,12 +167,21 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.insert_localized_app_metadata(apps) appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US') - self.assertTrue(os.path.isfile(os.path.join( - appdir, - 'icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png'))) - self.assertTrue(os.path.isfile(os.path.join( - appdir, - 'featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png'))) + self.assertTrue( + os.path.isfile( + os.path.join( + appdir, 'icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png' + ) + ) + ) + self.assertTrue( + os.path.isfile( + os.path.join( + appdir, + 'featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png', + ) + ) + ) self.assertEqual(6, len(apps)) for packageName, app in apps.items(): @@ -1894,7 +1903,10 @@ class UpdateTest(unittest.TestCase): with open('repo/index-v2.json') as fp: index = json.load(fp) self.assertEqual( - {'System': {'name': {'en-US': 'System Apps'}}, 'Time': {'name': {'en-US': 'Time'}}}, + { + 'System': {'name': {'en-US': 'System Apps'}}, + 'Time': {'name': {'en-US': 'Time'}}, + }, index['repo'][CATEGORIES_CONFIG_NAME], ) @@ -1923,28 +1935,40 @@ class UpdateTest(unittest.TestCase): ) def test_parse_ipa(self): - ipa_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa') + ipa_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'com.fake.IpaApp_1000000000001.ipa', + ) result = fdroidserver.update.parse_ipa(ipa_path, 'fake_size', 'fake_sha') self.maxDiff = None - self.assertDictEqual(result, { - 'apkName': 'com.fake.IpaApp_1000000000001.ipa', - 'hash': 'fake_sha', - 'hashType': 'sha256', - 'packageName': 'org.onionshare.OnionShare', - 'size': 'fake_size', - 'versionCode': 1000000000001, - 'versionName': '1.0.1', - }) + self.assertDictEqual( + result, + { + 'apkName': 'com.fake.IpaApp_1000000000001.ipa', + 'hash': 'fake_sha', + 'hashType': 'sha256', + 'packageName': 'org.onionshare.OnionShare', + 'size': 'fake_size', + 'versionCode': 1000000000001, + 'versionName': '1.0.1', + }, + ) class TestUpdateVersionStringToInt(unittest.TestCase): - def test_version_string_to_int(self): - self.assertEqual(fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003) + self.assertEqual( + fdroidserver.update.version_string_to_int("1.2.3"), 1000002000003 + ) self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0003"), 3) self.assertEqual(fdroidserver.update.version_string_to_int("0.0.0"), 0) - self.assertEqual(fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021) - self.assertEqual(fdroidserver.update.version_string_to_int("18446744.073709.551615"), 18446744073709551615) + self.assertEqual( + fdroidserver.update.version_string_to_int("4321.321.21"), 4321000321000021 + ) + self.assertEqual( + fdroidserver.update.version_string_to_int("18446744.073709.551615"), + 18446744073709551615, + ) def test_version_string_to_int_value_errors(self): with self.assertRaises(ValueError): @@ -1960,7 +1984,6 @@ class TestUpdateVersionStringToInt(unittest.TestCase): class TestScanRepoForIpas(unittest.TestCase): - def setUp(self): self.maxDiff = None @@ -1980,12 +2003,12 @@ class TestScanRepoForIpas(unittest.TestCase): def mocked_parse(p, s, c): # pylint: disable=unused-argument - return { - 'packageName': 'abc' if 'abc' in p else 'xyz' - } + return {'packageName': 'abc' if 'abc' in p else 'xyz'} with mock.patch('fdroidserver.update.parse_ipa', mocked_parse): - ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas(apkcache, repodir, knownapks) + ipas, checkchanged = fdroidserver.update.scan_repo_for_ipas( + apkcache, repodir, knownapks + ) self.assertEqual(checkchanged, True) self.assertEqual(len(ipas), 2) @@ -1993,14 +2016,22 @@ class TestScanRepoForIpas(unittest.TestCase): self.assertTrue('abc' in package_names_in_ipas) self.assertTrue('xyz' in package_names_in_ipas) - apkcache_setter_package_name = [x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls] + apkcache_setter_package_name = [ + x.args[1]['packageName'] for x in apkcache.__setitem__.mock_calls + ] self.assertTrue('abc' in apkcache_setter_package_name) self.assertTrue('xyz' in apkcache_setter_package_name) self.assertEqual(apkcache.__setitem__.call_count, 2) knownapks.recordapk.call_count = 2 - self.assertTrue(unittest.mock.call('abc.Def_123.ipa', 'abc') in knownapks.recordapk.mock_calls) - self.assertTrue(unittest.mock.call('xyz.XXX_123.ipa', 'xyz') in knownapks.recordapk.mock_calls) + self.assertTrue( + unittest.mock.call('abc.Def_123.ipa', 'abc') + in knownapks.recordapk.mock_calls + ) + self.assertTrue( + unittest.mock.call('xyz.XXX_123.ipa', 'xyz') + in knownapks.recordapk.mock_calls + ) if __name__ == "__main__": From dc7170e709b5ed1cfbb7051864ff8719e4f02afb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 11 Jan 2024 15:56:42 +0100 Subject: [PATCH 24/28] gitlab-ci: install biplist if available, otherwise skip test_parse_ipa Fedora does not have a biplist package. --- .gitlab-ci.yml | 12 ++++++++++-- tests/update.TestCase | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00fd097c..866a2ec8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -99,6 +99,7 @@ debian_testing: git gnupg ipfs-cid + python3-biplist python3-defusedxml python3-pycountry python3-setuptools @@ -124,7 +125,14 @@ ubuntu_lts_ppa: - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list - apt-get update - apt-get dist-upgrade - - apt-get install --install-recommends dexdump fdroidserver git python3-pycountry python3-setuptools sdkmanager + - apt-get install --install-recommends + dexdump + fdroidserver + git + python3-biplist + python3-pycountry + python3-setuptools + sdkmanager # Test things work with a default branch other than 'master' - git config --global init.defaultBranch thisisnotmasterormain @@ -350,7 +358,7 @@ macOS: - /bin/bash -n gradlew-fdroid tests/run-tests # TODO remove the packages below once they are included in the Homebrew package - - $(brew --prefix fdroidserver)/libexec/bin/python3 -m pip install pycountry + - $(brew --prefix fdroidserver)/libexec/bin/python3 -m pip install biplist pycountry # test fdroidserver from git with current package's dependencies - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests diff --git a/tests/update.TestCase b/tests/update.TestCase index cf8a222a..bebab3f0 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -1935,6 +1935,12 @@ class UpdateTest(unittest.TestCase): ) def test_parse_ipa(self): + try: + import biplist # Fedora does not have a biplist package + + biplist # silence the linters + except ImportError as e: + self.skipTest(str(e)) ipa_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'com.fake.IpaApp_1000000000001.ipa', From 3f50372d8d21cbce2eee0d75af301fc82976c877 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 Jan 2024 21:46:56 +0100 Subject: [PATCH 25/28] config: test cases for serverwebroot: with string and list --- tests/common.TestCase | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/common.TestCase b/tests/common.TestCase index 4d9ce009..68ceee55 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2868,6 +2868,22 @@ class CommonTest(unittest.TestCase): with self.assertRaises(TypeError): fdroidserver.common.config_type_check('config/mirrors.yml', str()) + def test_config_serverwebroot_str(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""") + self.assertEqual( + ['foo@example.com:/var/www/'], + fdroidserver.common.read_config()['serverwebroot'], + ) + + def test_config_serverwebroot_list(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot:\n - foo@example.com:/var/www""") + self.assertEqual( + ['foo@example.com:/var/www/'], + fdroidserver.common.read_config()['serverwebroot'], + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 7a656d45e3d26bb1888f7018a33847b04b967163 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 Jan 2024 21:58:12 +0100 Subject: [PATCH 26/28] config: convert serverwebroot: to list-of-dicts format This allows for more metadata about the server and deploy mode. --- examples/config.yml | 6 ++++++ fdroidserver/common.py | 13 +++++++++---- fdroidserver/deploy.py | 9 +++++---- tests/common.TestCase | 18 +++++++++++++----- tests/deploy.TestCase | 22 +++++++++++----------- 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 12f7d138..59453376 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -178,6 +178,12 @@ # serverwebroot: # - foo.com:/usr/share/nginx/www/fdroid # - bar.info:/var/www/fdroid +# +# There is a special mode to only deploy the index file: +# +# serverwebroot: +# - url: 'me@b.az:/srv/fdroid' +# indexOnly: true # When running fdroid processes on a remote server, it is possible to diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e97cedf5..3da0a193 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -462,18 +462,22 @@ def read_config(opts=None): if 'serverwebroot' in config: if isinstance(config['serverwebroot'], str): - roots = [config['serverwebroot']] + roots = [{'url': config['serverwebroot']}] elif all(isinstance(item, str) for item in config['serverwebroot']): + roots = [{'url': i} for i in config['serverwebroot']] + elif all(isinstance(item, dict) for item in config['serverwebroot']): roots = config['serverwebroot'] else: raise TypeError(_('only accepts strings, lists, and tuples')) rootlist = [] - for rootstr in roots: + for d in roots: # since this is used with rsync, where trailing slashes have # meaning, ensure there is always a trailing slash + rootstr = d['url'] if rootstr[-1] != '/': rootstr += '/' - rootlist.append(rootstr.replace('//', '/')) + d['url'] = rootstr.replace('//', '/') + rootlist.append(d) config['serverwebroot'] = rootlist if 'servergitmirrors' in config: @@ -4052,7 +4056,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None): logging.debug(_('skip deploying full build logs: not enabled in config')) return - for webroot in config.get('serverwebroot', []): + for d in config.get('serverwebroot', []): + webroot = d['url'] cmd = ['rsync', '--archive', '--delete-after', diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 7cc709d6..aef9205b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -294,11 +294,12 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] elif 'identity_file' in config: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) + url = serverwebroot['url'] + logging.info('rsyncing ' + repo_section + ' to ' + url) excludes = _get_index_excludes(repo_section) - if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: raise FDroidException() - if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: + if subprocess.call(rsyncargs + [repo_section, url]) != 0: raise FDroidException() # upload "current version" symlinks if requested if config['make_current_version_link'] and repo_section == 'repo': @@ -308,7 +309,7 @@ def update_serverwebroot(serverwebroot, repo_section): if os.path.islink(f): links_to_upload.append(f) if len(links_to_upload) > 0: - if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0: + if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: raise FDroidException() diff --git a/tests/common.TestCase b/tests/common.TestCase index 68ceee55..55202dcc 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options.quiet = False fdroidserver.common.config = {} fdroidserver.common.config['serverwebroot'] = [ - 'example.com:/var/www/fdroid/', - 'example.com:/var/www/fbot/', + {'url': 'example.com:/var/www/fdroid/'}, + {'url': 'example.com:/var/www/fbot/'}, ] fdroidserver.common.config['deploy_process_logs'] = True fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' @@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase): fdroidserver.common.options = mock.Mock() fdroidserver.common.config = {} - fdroidserver.common.config['serverwebroot'] = [fakeserver] + fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}] fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' def assert_subprocess_call(cmd): @@ -2872,7 +2872,7 @@ class CommonTest(unittest.TestCase): os.chdir(self.testdir) Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""") self.assertEqual( - ['foo@example.com:/var/www/'], + [{'url': 'foo@example.com:/var/www/'}], fdroidserver.common.read_config()['serverwebroot'], ) @@ -2880,7 +2880,15 @@ class CommonTest(unittest.TestCase): os.chdir(self.testdir) Path('config.yml').write_text("""serverwebroot:\n - foo@example.com:/var/www""") self.assertEqual( - ['foo@example.com:/var/www/'], + [{'url': 'foo@example.com:/var/www/'}], + fdroidserver.common.read_config()['serverwebroot'], + ) + + def test_config_serverwebroot_dict(self): + os.chdir(self.testdir) + Path('config.yml').write_text("""serverwebroot:\n - url: 'foo@example.com:/var/www'""") + self.assertEqual( + [{'url': 'foo@example.com:/var/www/'}], fdroidserver.common.read_config()['serverwebroot'], ) diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 5539af4c..b1f1f103 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -45,16 +45,16 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') - serverwebroot = Path('serverwebroot') - serverwebroot.mkdir() + url = Path('url') + url.mkdir() # setup parameters for this test run fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = False - dest_apk = Path(serverwebroot) / fake_apk + dest_apk = url / fake_apk self.assertFalse(dest_apk.is_file()) - fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo') + fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') self.assertTrue(dest_apk.is_file()) @mock.patch.dict(os.environ, clear=True) @@ -72,7 +72,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.quiet = True fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = True - serverwebroot = "example.com:/var/www/fdroid" + url = "example.com:/var/www/fdroid" repo_section = 'repo' # setup function for asserting subprocess.call invocations @@ -123,7 +123,7 @@ class DeployTest(unittest.TestCase): '--safe-links', '--quiet', 'repo', - serverwebroot, + url, ], ) elif call_iteration == 2: @@ -152,7 +152,7 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call') def test_update_serverwebroot_with_id_file(self): @@ -163,7 +163,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False - serverwebroot = "example.com:/var/www/fdroid" + url = "example.com:/var/www/fdroid" repo_section = 'archive' # setup function for asserting subprocess.call invocations @@ -204,7 +204,7 @@ class DeployTest(unittest.TestCase): '--exclude', 'archive/index.xml', 'archive', - serverwebroot, + url, ], ) elif call_iteration == 1: @@ -220,7 +220,7 @@ class DeployTest(unittest.TestCase): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], 'archive', - serverwebroot, + url, ], ) else: @@ -229,7 +229,7 @@ class DeployTest(unittest.TestCase): return 0 with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section) + fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') @unittest.skipIf( From fbf097d39083928e74ab043b1d8c00db079ad0d3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 23 Jan 2024 14:39:57 +0100 Subject: [PATCH 27/28] deploy: update_serverwebroot() works w/o options/config Since update_serverwebroot() is part of the public API, this function should work without setting `fdroidserver.deploy.options` or `fdroidserver.deploy.config`. --- fdroidserver/deploy.py | 12 ++++++------ tests/deploy.TestCase | 19 +++++++------------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index aef9205b..1cea0227 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -284,15 +284,15 @@ def update_serverwebroot(serverwebroot, repo_section): _('rsync is missing or broken: {error}').format(error=e) ) from e rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links'] - if not options.no_checksum: + if not options or not options.no_checksum: rsyncargs.append('--checksum') - if options.verbose: + if options and options.verbose: rsyncargs += ['--verbose'] - if options.quiet: + if options and options.quiet: rsyncargs += ['--quiet'] - if options.identity_file is not None: + if options and options.identity_file: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] - elif 'identity_file' in config: + elif config and config.get('identity_file'): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] url = serverwebroot['url'] logging.info('rsyncing ' + repo_section + ' to ' + url) @@ -302,7 +302,7 @@ def update_serverwebroot(serverwebroot, repo_section): if subprocess.call(rsyncargs + [repo_section, url]) != 0: raise FDroidException() # upload "current version" symlinks if requested - if config['make_current_version_link'] and repo_section == 'repo': + if config and config.get('make_current_version_link') and repo_section == 'repo': links_to_upload = [] for f in glob.glob('*.apk') \ + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index b1f1f103..7e6a9857 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -32,13 +32,11 @@ class DeployTest(unittest.TestCase): self._td = mkdtemp() self.testdir = self._td.name - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.config = {} - def tearDown(self): self._td.cleanup() def test_update_serverwebroot(self): + """rsync works with file paths, so this test uses paths for the URLs""" os.chdir(self.testdir) repo = Path('repo') repo.mkdir(parents=True) @@ -48,10 +46,6 @@ class DeployTest(unittest.TestCase): url = Path('url') url.mkdir() - # setup parameters for this test run - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = False - dest_apk = url / fake_apk self.assertFalse(dest_apk.is_file()) fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') @@ -66,12 +60,12 @@ class DeployTest(unittest.TestCase): def test_update_serverwebroot_make_cur_version_link(self): # setup parameters for this test run + fdroidserver.deploy.options = mock.Mock() fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['make_current_version_link'] = True + fdroidserver.deploy.config = {'make_current_version_link': True} url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -157,12 +151,13 @@ class DeployTest(unittest.TestCase): def test_update_serverwebroot_with_id_file(self): # setup parameters for this test run - fdroidserver.deploy.options.no_chcksum = False + fdroidserver.deploy.options = mock.Mock() + fdroidserver.deploy.options.identity_file = None + fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = True fdroidserver.deploy.options.quiet = False fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.config['identity_file'] = './id_rsa' - fdroidserver.deploy.config['make_current_version_link'] = False + fdroidserver.deploy.config = {'identity_file': './id_rsa'} url = "example.com:/var/www/fdroid" repo_section = 'archive' From 810387a009ddf0ad3f86fcb041b3487e52ef96bc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 23 Jan 2024 18:47:17 +0100 Subject: [PATCH 28/28] deploy: update_serverwebroots() for testable logic This moves all of the serverwebroot: logic into a function, and adds tests. I did this because I ran into issues in the logic in main(): Traceback (most recent call last): File "/builds/eighthave/fdroidserver/fdroid", line 22, in fdroidserver.__main__.main() File "/builds/eighthave/fdroidserver/fdroidserver/__main__.py", line 230, in main raise e File "/builds/eighthave/fdroidserver/fdroidserver/__main__.py", line 211, in main mod.main() File "/builds/eighthave/fdroidserver/fdroidserver/deploy.py", line 753, in main s = serverwebroot.rstrip('/').split(':') AttributeError: 'dict' object has no attribute 'rstrip' --- fdroidserver/__init__.py | 2 ++ fdroidserver/deploy.py | 52 ++++++++++++++++++++++++--------------- tests/deploy.TestCase | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/fdroidserver/__init__.py b/fdroidserver/__init__.py index ab9ab1bc..9e4c197f 100644 --- a/fdroidserver/__init__.py +++ b/fdroidserver/__init__.py @@ -55,7 +55,9 @@ scan_apk # NOQA: B101 scan_repo_files # NOQA: B101 from fdroidserver.deploy import (update_awsbucket, update_servergitmirrors, + update_serverwebroots, update_serverwebroot) # NOQA: E402 update_awsbucket # NOQA: B101 update_servergitmirrors # NOQA: B101 +update_serverwebroots # NOQA: B101 update_serverwebroot # NOQA: B101 diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 1cea0227..92287f1b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -313,6 +313,34 @@ def update_serverwebroot(serverwebroot, repo_section): raise FDroidException() +def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True): + for d in serverwebroots: + # this supports both an ssh host:path and just a path + serverwebroot = d['url'] + s = serverwebroot.rstrip('/').split(':') + if len(s) == 1: + fdroiddir = s[0] + elif len(s) == 2: + host, fdroiddir = s + else: + logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) + sys.exit(1) + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': + logging.error( + _( + 'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:' + ) + + '\n\t' + + serverwebroot.rstrip('/') + + '/fdroid\n\t' + + serverwebroot.rstrip('/').rstrip(repobase) + + 'fdroid' + ) + sys.exit(1) + update_serverwebroot(d, repo_section) + + def sync_from_localcopy(repo_section, local_copy_dir): """Sync the repo from "local copy dir" filesystem to this box. @@ -748,24 +776,6 @@ def main(): else: standardwebroot = True - for serverwebroot in config.get('serverwebroot', []): - # this supports both an ssh host:path and just a path - s = serverwebroot.rstrip('/').split(':') - if len(s) == 1: - fdroiddir = s[0] - elif len(s) == 2: - host, fdroiddir = s - else: - logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) - sys.exit(1) - repobase = os.path.basename(fdroiddir) - if standardwebroot and repobase != 'fdroid': - logging.error('serverwebroot path does not end with "fdroid", ' - + 'perhaps you meant one of these:\n\t' - + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') - sys.exit(1) - if options.local_copy_dir is not None: local_copy_dir = options.local_copy_dir elif config.get('local_copy_dir'): @@ -826,8 +836,10 @@ def main(): sync_from_localcopy(repo_section, local_copy_dir) else: update_localcopy(repo_section, local_copy_dir) - for serverwebroot in config.get('serverwebroot', []): - update_serverwebroot(serverwebroot, repo_section) + if config.get('serverwebroot'): + update_serverwebroots( + config['serverwebroot'], repo_section, standardwebroot + ) if config.get('servergitmirrors', []): # update_servergitmirrors will take care of multiple mirrors so don't need a foreach servergitmirrors = config.get('servergitmirrors', []) diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 7e6a9857..e4334725 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -35,6 +35,59 @@ class DeployTest(unittest.TestCase): def tearDown(self): self._td.cleanup() + def test_update_serverwebroots_bad_None(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(None, 'repo') + + def test_update_serverwebroots_bad_int(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(9, 'repo') + + def test_update_serverwebroots_bad_float(self): + with self.assertRaises(TypeError): + fdroidserver.deploy.update_serverwebroots(1.0, 'repo') + + def test_update_serverwebroots(self): + """rsync works with file paths, so this test uses paths for the URLs""" + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir() + fake_apk = repo / 'fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + url0 = Path('url0/fdroid') + url0.mkdir(parents=True) + url1 = Path('url1/fdroid') + url1.mkdir(parents=True) + + dest_apk0 = url0 / fake_apk + dest_apk1 = url1 / fake_apk + self.assertFalse(dest_apk0.is_file()) + self.assertFalse(dest_apk1.is_file()) + fdroidserver.deploy.update_serverwebroots( + [ + {'url': str(url0)}, + {'url': str(url1)}, + ], + str(repo), + ) + self.assertTrue(dest_apk0.is_file()) + self.assertTrue(dest_apk1.is_file()) + + def test_update_serverwebroots_url_does_not_end_with_fdroid(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo') + + def test_update_serverwebroots_bad_ssh_url(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots( + [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo' + ) + + def test_update_serverwebroots_unsupported_ssh_url(self): + with self.assertRaises(SystemExit): + fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') + def test_update_serverwebroot(self): """rsync works with file paths, so this test uses paths for the URLs""" os.chdir(self.testdir)