1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-07-04 16:30:12 +02:00

Merge branch 'sort-fdroid-build-all' into 'master'

sort `fdroid build --all` by most recent first

See merge request fdroid/fdroidserver!390
This commit is contained in:
Marcus 2017-12-02 12:34:27 +00:00
commit 167c730ad8
10 changed files with 163 additions and 116 deletions

View File

@ -1111,7 +1111,7 @@ def main():
# Read all app and srclib metadata
pkgs = common.read_pkg_args(options.appid, True)
allapps = metadata.read_metadata(not options.onserver, pkgs)
allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
apps = common.read_app_args(options.appid, allapps, True)
for appid, app in list(apps.items()):

View File

@ -26,6 +26,7 @@ import logging
import textwrap
import io
import yaml
from collections import OrderedDict
# use libyaml if it is available
try:
from yaml import CLoader
@ -703,35 +704,50 @@ def read_srclibs():
srclibs[srclibname] = parse_srclib(metadatapath)
def read_metadata(xref=True, check_vcs=[]):
"""
Read all metadata. Returns a list of 'app' objects (which are dictionaries as
returned by the parse_txt_metadata function.
def read_metadata(xref=True, check_vcs=[], sort_by_time=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
builds a list of App instances from those files. The list is
sorted based on creation time, newest first. Most of the time,
the newer files are the most interesting.
If there are multiple metadata files for a single appid, then the first
file that is parsed wins over all the others, and the rest throw an
exception. So the original .txt format is parsed first, at least until
newer formats stabilize.
check_vcs is the list of packageNames to check for .fdroid.yml in source
"""
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
read_srclibs()
apps = {}
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
# If there are multiple metadata files for a single appid, then the first
# file that is parsed wins over all the others, and the rest throw an
# exception. So the original .txt format is parsed first, at least until
# newer formats stabilize.
metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
+ glob.glob(os.path.join('metadata', '*.json'))
+ glob.glob(os.path.join('metadata', '*.yml'))
+ glob.glob('.fdroid.txt')
+ glob.glob('.fdroid.json')
+ glob.glob('.fdroid.yml'))
for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
+ glob.glob(os.path.join('metadata', '*.json'))
+ glob.glob(os.path.join('metadata', '*.yml'))
+ glob.glob('.fdroid.txt')
+ glob.glob('.fdroid.json')
+ glob.glob('.fdroid.yml')):
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
else:
# most things want the index alpha sorted for stability
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
if packageName in apps:
warn_or_exception(_("Found multiple metadata files for {appid}")

View File

@ -3,6 +3,7 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging
import optparse
import os
import re
@ -46,22 +47,27 @@ class BuildTest(unittest.TestCase):
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.isfile(path))
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
def test_force_gradle_build_tools(self):
testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
if not os.path.exists(testsbase):
os.makedirs(testsbase)
testsdir = tempfile.mkdtemp(prefix='test_adapt_gradle', dir=testsbase)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'),
os.path.join(testsdir, 'source-files'))
os.path.join(testdir, 'source-files'))
teststring = 'FAKE_VERSION_FOR_TESTING'
fdroidserver.build.force_gradle_build_tools(testsdir, teststring)
fdroidserver.build.force_gradle_build_tools(testdir, teststring)
pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
for p in ('source-files/fdroid/fdroidclient/build.gradle',
'source-files/Zillode/syncthing-silk/build.gradle',
'source-files/open-keychain/open-keychain/build.gradle',
'source-files/osmandapp/osmand/build.gradle',
'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
with open(os.path.join(testsdir, p), 'rb') as f:
with open(os.path.join(testdir, p), 'rb') as f:
filedata = f.read()
self.assertIsNotNone(pattern.search(filedata))

View File

@ -96,8 +96,8 @@ class CommonTest(unittest.TestCase):
print('no build-tools found: ' + build_tools)
def test_find_java_root_path(self):
tmptestsdir = tempfile.mkdtemp(prefix='test_find_java_root_path', dir=self.tmpdir)
os.chdir(tmptestsdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
all_pathlists = [
([ # Debian
@ -170,11 +170,11 @@ class CommonTest(unittest.TestCase):
testint = 99999999
teststr = 'FAKE_STR_FOR_TESTING'
tmptestsdir = tempfile.mkdtemp(prefix='test_prepare_sources', dir=self.tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
shutil.copytree(os.path.join(self.basedir, 'source-files'),
os.path.join(tmptestsdir, 'source-files'))
os.path.join(testdir, 'source-files'))
testdir = os.path.join(tmptestsdir, 'source-files', 'fdroid', 'fdroidclient')
fdroidclient_testdir = os.path.join(testdir, 'source-files', 'fdroid', 'fdroidclient')
config = dict()
config['sdk_path'] = os.getenv('ANDROID_HOME')
@ -201,13 +201,14 @@ class CommonTest(unittest.TestCase):
def getsrclib(self):
return None
fdroidserver.common.prepare_source(FakeVcs(), app, build, testdir, testdir, testdir)
fdroidserver.common.prepare_source(FakeVcs(), app, build,
fdroidclient_testdir, fdroidclient_testdir, fdroidclient_testdir)
with open(os.path.join(testdir, 'build.gradle'), 'r') as f:
with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f:
filedata = f.read()
self.assertIsNotNone(re.search("\s+compileSdkVersion %s\s+" % testint, filedata))
with open(os.path.join(testdir, 'AndroidManifest.xml')) as f:
with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f:
filedata = f.read()
self.assertIsNone(re.search('android:debuggable', filedata))
self.assertIsNotNone(re.search('android:versionName="%s"' % build.versionName, filedata))
@ -215,7 +216,7 @@ class CommonTest(unittest.TestCase):
def test_prepare_sources_refresh(self):
packageName = 'org.fdroid.ci.test.app'
testdir = tempfile.mkdtemp(prefix='test_prepare_sources_refresh', dir=self.tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
print('testdir', testdir)
os.chdir(testdir)
os.mkdir('build')
@ -262,7 +263,7 @@ class CommonTest(unittest.TestCase):
fdroidserver.signindex.config = config
sourcedir = os.path.join(self.basedir, 'signindex')
testsdir = tempfile.mkdtemp(prefix='test_signjar', dir=self.tmpdir)
testsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
for f in ('testy.jar', 'guardianproject.jar',):
sourcefile = os.path.join(sourcedir, f)
testfile = os.path.join(testsdir, f)
@ -291,7 +292,7 @@ class CommonTest(unittest.TestCase):
sourceapk = os.path.join(self.basedir, 'urzip.apk')
testdir = tempfile.mkdtemp(prefix='test_verify_apks', dir=self.tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
print('testdir', testdir)
copyapk = os.path.join(testdir, 'urzip-copy.apk')
@ -529,4 +530,4 @@ if __name__ == "__main__":
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(CommonTest))
unittest.main(failfast=True)
unittest.main(failfast=False)

View File

@ -3,6 +3,7 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging
import optparse
import os
import requests
@ -24,8 +25,15 @@ import import_proxy
class ImportTest(unittest.TestCase):
'''fdroid import'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
def test_import_gitlab(self):
os.chdir(os.path.dirname(__file__))
# FDroidPopen needs some config to work
config = dict()
fdroidserver.common.fill_config_defaults(config)

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import inspect
import logging
import optparse
import os
import sys
@ -31,30 +32,33 @@ GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A61
class IndexTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
fdroidserver.common.config = None
config = fdroidserver.common.read_config(fdroidserver.common.options)
config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
fdroidserver.common.config = config
fdroidserver.signindex.config = config
@staticmethod
def test_verify_jar_signature_succeeds():
basedir = os.path.dirname(__file__)
source_dir = os.path.join(basedir, 'signindex')
def test_verify_jar_signature_succeeds(self):
source_dir = os.path.join(self.basedir, 'signindex')
for f in ('testy.jar', 'guardianproject.jar'):
testfile = os.path.join(source_dir, f)
fdroidserver.common.verify_jar_signature(testfile)
def test_verify_jar_signature_fails(self):
basedir = os.path.dirname(__file__)
source_dir = os.path.join(basedir, 'signindex')
source_dir = os.path.join(self.basedir, 'signindex')
testfile = os.path.join(source_dir, 'unsigned.jar')
with self.assertRaises(fdroidserver.index.VerificationException):
fdroidserver.common.verify_jar_signature(testfile)
def test_get_public_key_from_jar_succeeds(self):
basedir = os.path.dirname(__file__)
source_dir = os.path.join(basedir, 'signindex')
source_dir = os.path.join(self.basedir, 'signindex')
for f in ('testy.jar', 'guardianproject.jar'):
testfile = os.path.join(source_dir, f)
jar = zipfile.ZipFile(testfile)
@ -68,8 +72,7 @@ class IndexTest(unittest.TestCase):
self.assertTrue(fingerprint == GP_FINGERPRINT)
def test_get_public_key_from_jar_fails(self):
basedir = os.path.dirname(__file__)
source_dir = os.path.join(basedir, 'signindex')
source_dir = os.path.join(self.basedir, 'signindex')
testfile = os.path.join(source_dir, 'unsigned.jar')
jar = zipfile.ZipFile(testfile)
with self.assertRaises(fdroidserver.index.VerificationException):
@ -226,9 +229,6 @@ class IndexTest(unittest.TestCase):
if __name__ == "__main__":
if os.path.basename(os.getcwd()) != 'tests' and os.path.isdir('tests'):
os.chdir('tests')
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")

View File

@ -32,8 +32,7 @@ class LintTest(unittest.TestCase):
self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files())
tmpdir = os.path.join(localmodule, '.testfiles')
tmptestsdir = tempfile.mkdtemp(prefix='test_check_for_unsupported_metadata_files-',
dir=tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=tmpdir)
self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/'))
shutil.copytree(os.path.join(localmodule, 'tests', 'metadata'),
os.path.join(tmptestsdir, 'metadata'),

View File

@ -2,9 +2,13 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import glob
import inspect
import logging
import optparse
import os
import random
import shutil
import sys
import unittest
import yaml
@ -23,15 +27,20 @@ import fdroidserver.metadata
class MetadataTest(unittest.TestCase):
'''fdroidserver/metadata.py'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
def test_read_metadata(self):
def _build_yaml_representer(dumper, data):
'''Creates a YAML representation of a Build instance'''
return dumper.represent_dict(data)
testsdir = os.path.dirname(__file__)
os.chdir(testsdir)
self.maxDiff = None
# these need to be set to prevent code running on None, only
@ -58,12 +67,7 @@ class MetadataTest(unittest.TestCase):
# yaml.dump(frommeta, f, default_flow_style=False)
def test_rewrite_yaml_fakeotaupdate(self):
# setup/reset test dir if necessary and setup params
tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
testdir = tempfile.mkdtemp(prefix='test_rewrite_metadata_', dir=tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}
# rewrite metadata
@ -79,12 +83,7 @@ class MetadataTest(unittest.TestCase):
self.assertEqual(result.read(), orig.read())
def test_rewrite_yaml_fdroidclient(self):
# setup/reset test dir if necessary and setup params
tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
testdir = tempfile.mkdtemp(prefix='test_rewrite_metadata_', dir=tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}
# rewrite metadata
@ -100,12 +99,7 @@ class MetadataTest(unittest.TestCase):
self.assertEqual(result.read(), orig.read())
def test_rewrite_yaml_special_build_params(self):
# setup/reset test dir if necessary and setup params
tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
testdir = tempfile.mkdtemp(prefix='test_rewrite_metadata_', dir=tmpdir)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}
# rewrite metadata
@ -120,6 +114,31 @@ class MetadataTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(result.read(), orig.read())
def test_read_metadata_sort_by_time(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
metadatadir = os.path.join(testdir, 'metadata')
os.makedirs(metadatadir)
fdroidserver.common.config = {'accepted_formats': ['txt']}
randomlist = []
randomapps = glob.glob(os.path.join(self.basedir, 'metadata', '*.txt'))
random.shuffle(randomapps)
i = 1
for f in randomapps:
shutil.copy(f, metadatadir)
new = os.path.join(metadatadir, os.path.basename(f))
stat = os.stat(new)
os.utime(new, (stat.st_ctime, stat.st_mtime + i))
# prepend new item so newest is always first
randomlist = [os.path.basename(f)[:-4]] + randomlist
i += 1
os.chdir(testdir)
allapps = fdroidserver.metadata.read_metadata(xref=True, sort_by_time=True)
allappids = []
for appid, app in allapps.items():
allappids.append(appid)
self.assertEqual(randomlist, allappids)
if __name__ == "__main__":
parser = optparse.OptionParser()

View File

@ -11,6 +11,7 @@
#
import inspect
import logging
import optparse
import os
import sys
@ -32,6 +33,14 @@ from fdroidserver.exception import FDroidException
class PublishTest(unittest.TestCase):
'''fdroidserver/publish.py'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
def test_key_alias(self):
publish.config = {}
self.assertEqual('a163ec9b', publish.key_alias('com.example.app'))
@ -77,39 +86,35 @@ class PublishTest(unittest.TestCase):
'com.example.anotherapp',
'org.org.org']
with tempfile.TemporaryDirectory() as tmpdir:
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
with open('config.py', 'w') as f:
pass
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
with open('config.py', 'w') as f:
pass
publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2)
self.maxDiff = None
expected = {
"com.example.anotherapp": {
"signer": "fa3f6a017541ee7fe797be084b1bcfbf92418a7589ef1f7fdeb46741b6d2e9c3"
},
"com.example.app": {
"signer": "d34f678afbaa8f2fa6cc0edd6f0c2d1d2e2e9eb08bea521b24c740806016bff4"
},
"org.org.org": {
"signer": "277655a6235bc6b0ef2d824396c51ba947f5ebc738c293d887e7083ff338af82"
},
"org.test.testy": {
"signer": "6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c"
}
}
self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
self.maxDiff = None
expected = {
"com.example.anotherapp": {
"signer": "fa3f6a017541ee7fe797be084b1bcfbf92418a7589ef1f7fdeb46741b6d2e9c3"
},
"com.example.app": {
"signer": "d34f678afbaa8f2fa6cc0edd6f0c2d1d2e2e9eb08bea521b24c740806016bff4"
},
"org.org.org": {
"signer": "277655a6235bc6b0ef2d824396c51ba947f5ebc738c293d887e7083ff338af82"
},
"org.test.testy": {
"signer": "6ae5355157a47ddcc3834a71f57f6fb5a8c2621c8e0dc739e9ddf59f865e497c"
}
}
self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints())
with open('config.py', 'r') as f:
self.assertEqual(textwrap.dedent('''\
with open('config.py', 'r') as f:
self.assertEqual(textwrap.dedent('''\
repo_key_sha256 = "c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41"
'''), f.read())
finally:
os.chdir(orig_cwd)
repo_key_sha256 = "c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41"
'''), f.read())
def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self):
common.config = {}
@ -122,21 +127,14 @@ class PublishTest(unittest.TestCase):
publish.config['repo_keyalias'] = 'repokey'
publish.config['repo_key_sha256'] = 'bad bad bad bad bad bad bad bad bad bad bad bad'
with tempfile.TemporaryDirectory() as tmpdir:
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
publish.store_stats_fdroid_signing_key_fingerprints({}, indent=2)
with self.assertRaises(FDroidException):
common.load_stats_fdroid_signing_key_fingerprints()
finally:
os.chdir(orig_cwd)
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
publish.store_stats_fdroid_signing_key_fingerprints({}, indent=2)
with self.assertRaises(FDroidException):
common.load_stats_fdroid_signing_key_fingerprints()
if __name__ == "__main__":
if os.path.basename(os.getcwd()) != 'tests' and os.path.isdir('tests'):
os.chdir('tests')
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")

View File

@ -87,7 +87,7 @@ class UpdateTest(unittest.TestCase):
tmpdir = os.path.join(localmodule, '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix='test_insert_triple_t_metadata-', dir=tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=tmpdir)
packageDir = os.path.join(tmptestsdir, 'build', packageName)
shutil.copytree(importer, packageDir)
@ -374,7 +374,7 @@ class UpdateTest(unittest.TestCase):
tmpdir = os.path.join(localmodule, '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix='test_process_apk_signed_by_disabled_algorithms-',
tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name,
dir=tmpdir)
print('tmptestsdir', tmptestsdir)
os.chdir(tmptestsdir)
@ -504,7 +504,7 @@ class UpdateTest(unittest.TestCase):
tmpdir = os.path.join(localmodule, '.testfiles')
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
tmptestsdir = tempfile.mkdtemp(prefix='test_create_metadata_from_template-',
tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name,
dir=tmpdir)
print('tmptestsdir', tmptestsdir)
os.chdir(tmptestsdir)