1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-04 14:30:11 +01:00
fdroidserver/tests/index.TestCase
2023-02-16 21:45:39 +01:00

566 lines
24 KiB
Python
Executable File

#!/usr/bin/env python3
import datetime
import inspect
import logging
import optparse
import os
import sys
import unittest
import zipfile
from unittest.mock import patch
import requests
import tempfile
import json
import shutil
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
import fdroidserver.common
import fdroidserver.index
import fdroidserver.metadata
import fdroidserver.net
import fdroidserver.signindex
import fdroidserver.publish
from testcommon import TmpCwd, mkdtemp
from pathlib import Path
GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
class Options:
nosign = True
pretty = False
verbose = False
class IndexTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
os.chmod(os.path.join(self.basedir, 'config.py'), 0o600)
fdroidserver.common.config = None
fdroidserver.common.options = Options
config = fdroidserver.common.read_config(fdroidserver.common.options)
config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
fdroidserver.common.config = config
fdroidserver.signindex.config = config
fdroidserver.update.config = config
if not os.path.exists('repo/index-v1.jar'):
fdroidserver.signindex.sign_index(
os.path.join(self.basedir, 'repo'), 'index-v1.json'
)
self._td = mkdtemp()
self.testdir = self._td.name
def tearDown(self):
os.chdir(self.basedir)
self._td.cleanup()
os.remove('repo/index-v1.jar')
def test_get_public_key_from_jar_succeeds(self):
source_dir = os.path.join(self.basedir, 'signindex')
for f in ('testy.jar', 'guardianproject.jar'):
testfile = os.path.join(source_dir, f)
jar = zipfile.ZipFile(testfile)
_, fingerprint = fdroidserver.index.get_public_key_from_jar(jar)
# comparing fingerprints should be sufficient
if f == 'testy.jar':
self.assertEqual(
fingerprint,
'818E469465F96B704E27BE2FEE4C63AB'
+ '9F83DDF30E7A34C7371A4728D83B0BC1',
)
if f == 'guardianproject.jar':
self.assertTrue(fingerprint == GP_FINGERPRINT)
def test_get_public_key_from_jar_fails(self):
source_dir = os.path.join(self.basedir, 'signindex')
testfile = os.path.join(source_dir, 'unsigned.jar')
jar = zipfile.ZipFile(testfile)
with self.assertRaises(fdroidserver.index.VerificationException):
fdroidserver.index.get_public_key_from_jar(jar)
def test_download_repo_index_no_fingerprint(self):
with self.assertRaises(fdroidserver.index.VerificationException):
fdroidserver.index.download_repo_index("http://example.org")
def test_download_repo_index_no_jar(self):
with self.assertRaises(requests.exceptions.RequestException):
fdroidserver.index.download_repo_index(
"http://example.org?fingerprint=nope"
)
def test_get_repo_key_fingerprint(self):
pubkey, fingerprint = fdroidserver.index.extract_pubkey()
data, public_key, public_key_fingerprint = fdroidserver.index.get_index_from_jar(
'repo/index-v1.jar', fingerprint
)
self.assertIsNotNone(data)
self.assertIsNotNone(public_key)
self.assertIsNotNone(public_key_fingerprint)
def test_get_index_from_jar_with_bad_fingerprint(self):
pubkey, fingerprint = fdroidserver.index.extract_pubkey()
fingerprint = fingerprint[:-1] + 'G'
with self.assertRaises(fdroidserver.exception.VerificationException):
fdroidserver.index.get_index_from_jar('repo/index-v1.jar', fingerprint)
def test_get_index_from_jar_with_chars_to_be_stripped(self):
fingerprint = 'NOOOO F4 9A F3 F1 1E FD DF 20 DF FD 70 F5 E3 11 7B 99 76 67 41 67 AD CA 28 0E 6B 19 32 A0 60 1B 26 F6'
data, public_key, public_key_fingerprint = fdroidserver.index.get_index_from_jar(
'repo/index-v1.jar', fingerprint
)
@patch('requests.head')
def test_download_repo_index_same_etag(self, head):
url = 'http://example.org?fingerprint=test'
etag = '"4de5-54d840ce95cb9"'
head.return_value.headers = {'ETag': etag}
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
self.assertIsNone(index)
self.assertEqual(etag, new_etag)
@patch('requests.get')
@patch('requests.head')
def test_download_repo_index_new_etag(self, head, get):
url = 'http://example.org?fingerprint=' + GP_FINGERPRINT
etag = '"4de5-54d840ce95cb9"'
# fake HTTP answers
head.return_value.headers = {'ETag': 'new_etag'}
get.return_value.headers = {'ETag': 'new_etag'}
get.return_value.status_code = 200
testfile = os.path.join('signindex', 'guardianproject-v1.jar')
with open(testfile, 'rb') as file:
get.return_value.content = file.read()
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
# assert that the index was retrieved properly
self.assertEqual('Guardian Project Official Releases', index['repo']['name'])
self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint'])
self.assertTrue(len(index['repo']['pubkey']) > 500)
self.assertEqual(10, len(index['apps']))
self.assertEqual(10, len(index['packages']))
self.assertEqual('new_etag', new_etag)
@patch('fdroidserver.net.http_get')
def test_download_repo_index_url_parsing(self, mock_http_get):
mock_http_get.side_effect = lambda url, etag, timeout: (None, url)
repo_url = 'https://example.org/fdroid/repo'
index_url = 'https://example.org/fdroid/repo/index-v1.jar'
fingerprint_url = 'https://example.org/fdroid/repo?fingerprint=' + GP_FINGERPRINT
slash_url = 'https://example.org/fdroid/repo//?fingerprint=' + GP_FINGERPRINT
for url in (repo_url, index_url, fingerprint_url, slash_url):
_ignored, returned_url = fdroidserver.index.download_repo_index(url, verify_fingerprint=False)
self.assertEqual(index_url, returned_url)
def test_v1_sort_packages(self):
i = [{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_134.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 134},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_134_b30bb97.apk',
'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
'versionCode': 134},
{'packageName': 'b075b32b4ef1e8a869e00edb136bd48e34a0382b85ced8628f164d1199584e4e'},
{'packageName': '43af70d1aca437c2f9974c4634cc5abe45bdc4d5d71529ac4e553488d3bb3ff6'},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_135_b30bb97.apk',
'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
'versionCode': 135},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_135.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 135},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_133.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 133},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-weird-version.apk',
'signer': '99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff',
'versionCode': 133},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-custom.apk',
'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
'versionCode': 133},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-new-custom.apk',
'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
'versionCode': 135}]
o = [{'packageName': '43af70d1aca437c2f9974c4634cc5abe45bdc4d5d71529ac4e553488d3bb3ff6'},
{'packageName': 'b075b32b4ef1e8a869e00edb136bd48e34a0382b85ced8628f164d1199584e4e'},
# app test data
# # packages with reproducible developer signature
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_135_b30bb97.apk',
'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
'versionCode': 135},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_134_b30bb97.apk',
'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
'versionCode': 134},
# # packages build and signed by fdroid
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_135.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 135},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_134.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 134},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'org.smssecure.smssecure_133.apk',
'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
'versionCode': 133},
# # packages signed with unkown keys
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-new-custom.apk',
'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
'versionCode': 135},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-custom.apk',
'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
'versionCode': 133},
{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-weird-version.apk',
'signer': '99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff',
'versionCode': 133}]
fdroidserver.common.config = {}
fdroidserver.common.fill_config_defaults(fdroidserver.common.config)
fdroidserver.publish.config = fdroidserver.common.config
fdroidserver.publish.config['keystorepass'] = '123456'
fdroidserver.publish.config['keypass'] = '123456'
fdroidserver.publish.config['keystore'] = os.path.join(os.getcwd(),
'dummy-keystore.jks')
fdroidserver.publish.config['repo_keyalias'] = 'repokey'
testsmetadir = os.path.join(os.getcwd(), 'metadata')
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
shutil.copytree(testsmetadir, 'metadata')
sigkeyfps = {
"org.smssecure.smssecure": {
"signer": "b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6"
}
}
os.makedirs('stats')
jarfile = 'stats/publishsigkeys.jar'
with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
fdroidserver.publish.sign_sig_key_fingerprint_list(jarfile)
with open('config.py', 'w'):
pass
fdroidserver.index.v1_sort_packages(
i, fdroidserver.common.load_stats_fdroid_signing_key_fingerprints()
)
self.maxDiff = None
self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
def test_make_v0_repo_only(self):
os.chdir(self.testdir)
os.mkdir('repo')
repo_icons_dir = os.path.join('repo', 'icons')
self.assertFalse(os.path.isdir(repo_icons_dir))
repodict = {
'address': 'https://example.com/fdroid/repo',
'description': 'This is just a test',
'icon': 'blahblah',
'name': 'test',
'timestamp': datetime.datetime.now(),
'version': 12,
}
requestsdict = {'install': [], 'uninstall': []}
fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, {})
self.assertTrue(os.path.isdir(repo_icons_dir))
self.assertTrue(
os.path.exists(
os.path.join(
repo_icons_dir, fdroidserver.common.default_config['repo_icon']
)
)
)
self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
def test_make_v0(self):
os.chdir(self.testdir)
os.mkdir('metadata')
os.mkdir('repo')
metadatafile = 'metadata/info.zwanenburg.caffeinetile.yml'
shutil.copy(os.path.join(self.basedir, metadatafile), metadatafile)
repo_icons_dir = os.path.join('repo', 'icons')
self.assertFalse(os.path.isdir(repo_icons_dir))
repodict = {
'address': 'https://example.com/fdroid/repo',
'description': 'This is just a test',
'icon': 'blahblah',
'name': 'test',
'timestamp': datetime.datetime.now(),
'version': 12,
}
app = fdroidserver.metadata.parse_metadata(metadatafile)
app['icon'] = 'info.zwanenburg.caffeinetile.4.xml'
app['CurrentVersionCode'] = 4
apps = {app.id: app}
apk = {
'hash': 'dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db',
'hashType': 'sha256',
'uses-permission': [['android.permission.WAKE_LOCK', None]],
'uses-permission-sdk-23': [],
'features': [],
'icons_src': {
'160': 'res/drawable/ic_coffee_on.xml',
'-1': 'res/drawable/ic_coffee_on.xml',
},
'icons': {'160': 'info.zwanenburg.caffeinetile.4.xml'},
'antiFeatures': [],
'packageName': 'info.zwanenburg.caffeinetile',
'versionCode': 4,
'name': 'Caffeine Tile',
'versionName': '1.3',
'minSdkVersion': 24,
'targetSdkVersion': 25,
'sig': '03f9b2f848d22fd1d8d1331e8b1b486d',
'signer': '51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e',
'size': 11740,
'apkName': 'info.zwanenburg.caffeinetile_4.apk',
'icon': 'info.zwanenburg.caffeinetile.4.xml',
'added': datetime.datetime.fromtimestamp(1539122400),
}
requestsdict = {'install': [], 'uninstall': []}
fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
fdroidserver.common.config['make_current_version_link'] = True
fdroidserver.index.make_v0(apps, [apk], 'repo', repodict, requestsdict, {})
self.assertTrue(os.path.isdir(repo_icons_dir))
self.assertTrue(
os.path.exists(
os.path.join(
repo_icons_dir, fdroidserver.common.default_config['repo_icon']
)
)
)
self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
def test_v0_invalid_config_exception(self):
"""Index v0 needs additional config values when using --nosign
index.xml aka Index v0 includes the full repo public key in
the XML itself. So when running `fdroid update --nosign`,
there needs to be either repo_pubkey or a full keystore config
present.
"""
os.chdir(self.testdir)
os.mkdir('repo')
repo_icons_dir = os.path.join('repo', 'icons')
self.assertFalse(os.path.isdir(repo_icons_dir))
repodict = {
'address': 'https://example.com/fdroid/repo',
'description': 'This is just a test',
'icon': 'blahblah',
'name': 'test',
'timestamp': datetime.datetime.now(),
'version': 12,
}
requestsdict = {'install': [], 'uninstall': []}
fdroidserver.common.options.nosign = False
with self.assertRaises(fdroidserver.exception.FDroidException):
fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, {})
fdroidserver.common.options.nosign = True
with self.assertRaises(fdroidserver.exception.FDroidException):
fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, {})
fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
self.assertFalse(os.path.exists(os.path.join('repo', 'index.xml')))
self.assertFalse(os.path.exists(os.path.join('repo', 'index_unsigned.jar')))
self.assertFalse(os.path.exists(os.path.join('repo', 'index.jar')))
fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, {})
self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
self.assertTrue(os.path.exists(os.path.join('repo', 'index_unsigned.jar')))
self.assertFalse(os.path.exists(os.path.join('repo', 'index.jar')))
def test_github_get_mirror_service_urls(self):
for url in [
'git@github.com:foo/bar',
'git@github.com:foo/bar.git',
'https://github.com/foo/bar',
'https://github.com/foo/bar.git',
]:
self.assertEqual(
['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
fdroidserver.index.get_mirror_service_urls(url),
)
@patch.dict(os.environ, clear=True)
def test_gitlab_get_mirror_service_urls(self):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
git_mirror_path = Path('git-mirror/fdroid')
git_mirror_path.mkdir(parents=True)
ci_job_id = '12345678'
artifacts_url = (
'https://group.gitlab.io/-/project/-/jobs/%s/artifacts/public/fdroid'
% ci_job_id
)
with (git_mirror_path / 'placeholder').open('w') as fp:
fp.write(' ')
for url in [
'git@gitlab.com:group/project',
'git@gitlab.com:group/project.git',
'https://gitlab.com/group/project',
'https://gitlab.com/group/project.git',
]:
with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 1000):
expected = [
'https://group.gitlab.io/project/fdroid',
'https://gitlab.com/group/project/-/raw/master/fdroid',
]
self.assertEqual(
expected,
fdroidserver.index.get_mirror_service_urls(url),
)
with patch.dict(os.environ, clear=True):
os.environ['CI_JOB_ID'] = ci_job_id
self.assertEqual(
expected + [artifacts_url],
fdroidserver.index.get_mirror_service_urls(url),
)
with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10):
expected = [
'https://gitlab.com/group/project/-/raw/master/fdroid',
]
self.assertEqual(
expected,
fdroidserver.index.get_mirror_service_urls(url),
)
with patch.dict(os.environ, clear=True):
os.environ['CI_JOB_ID'] = ci_job_id
self.assertEqual(
expected + [artifacts_url],
fdroidserver.index.get_mirror_service_urls(url),
)
def test_make_website(self):
os.chdir(self.testdir)
os.mkdir('metadata')
os.mkdir('repo')
repodict = {
'address': 'https://example.com/fdroid/repo',
'description': 'This is just a test',
'icon': 'blahblah',
'name': 'test',
'timestamp': datetime.datetime.now(),
'version': 12,
}
fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
fdroidserver.index.make_website([], "repo", repodict)
self.assertTrue(os.path.exists(os.path.join('repo', 'index.html')))
self.assertTrue(os.path.exists(os.path.join('repo', 'index.css')))
self.assertTrue(os.path.exists(os.path.join('repo', 'index.png')))
try:
from html5print import CSSBeautifier, HTMLBeautifier
except ImportError:
print('WARNING: skipping rest of test since html5print is missing!')
return
with open(os.path.join("repo", "index.html")) as f:
html = f.read()
pretty_html = HTMLBeautifier.beautify(html)
self.maxDiff = None
self.assertEquals(html, pretty_html)
with open(os.path.join("repo", "index.css")) as f:
css = f.read()
pretty_css = CSSBeautifier.beautify(css)
self.maxDiff = None
self.assertEquals(css, pretty_css)
def test_v1_sort_packages_with_invalid(self):
i = [
{
'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-custom.fake',
'signer': None,
'versionCode': 11111,
}
]
fdroidserver.index.v1_sort_packages(
i, fdroidserver.common.load_stats_fdroid_signing_key_fingerprints()
)
def test_package_metadata(self):
"""A smoke check and format check of index.package_metadata()"""
def _kn(key):
return key[0].lower() + key[1:]
apps = fdroidserver.metadata.read_metadata()
fdroidserver.update.insert_localized_app_metadata(apps)
# smoke check all metadata files
for appid, app in apps.items():
metadata = fdroidserver.index.package_metadata(app, 'repo')
for k in ('Description', 'Name', 'Summary', 'video'):
if app.get(k):
self.assertTrue(isinstance(metadata[_kn(k)], dict))
for k in ('AuthorWebSite', 'IssueTracker', 'Translation', 'WebSite'):
if app.get(k):
self.assertTrue(isinstance(metadata[_kn(k)], str))
# make sure these known values were properly parsed and included
appid = 'info.guardianproject.urzip'
app = apps[appid]
metadata = fdroidserver.index.package_metadata(app, 'repo')
# files
self.assertEqual(36027, metadata['featureGraphic']['en-US']['size'])
self.assertEqual(1413, metadata['icon']['en-US']['size'])
# localized strings
self.assertEqual({'en-US': 'title'}, metadata['name'])
self.assertEqual({'en-US': 'video'}, metadata['video'])
# strings
self.assertEqual(
'https://dev.guardianproject.info/projects/urzip',
metadata['webSite'],
)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
parser = optparse.OptionParser()
parser.add_option(
"-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal",
)
(options, args) = parser.parse_args()
Options.verbose = options.verbose
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(IndexTest))
unittest.main(failfast=False)