mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-04 14:30:11 +01:00
Merge branch 'test-updates-and-related-bug-fixes' into 'master'
Test updates and related bug fixes I just set up some big tests of generating repos based on feeding as many random APKs into `fdroid update` as possible. On our jenkins server, the tests copy all of the APKs that the jenkins server has generated and builds a repo from them. This process caught lots of little glitches in the whole process. While these little glitches are usually caused by problematic APKs, `fdroid update` should handle them gracefully. Hopefully this set of fixes accomplishes that.
This commit is contained in:
Ciaran Gultnieks
commit
b86bfb94fb
@ -128,8 +128,8 @@ __complete_install() {
|
||||
|
||||
__complete_update() {
|
||||
opts="-h -c -v -q -b -i -I -e -w"
|
||||
lopts="--help --createmeta --verbose --quiet --buildreport --interactive
|
||||
--icons --editor --wiki --pretty --clean --delete-unknown"
|
||||
lopts="--help --create-metadata --verbose --quiet --buildreport
|
||||
--interactive --icons --editor --wiki --pretty --clean --delete-unknown"
|
||||
case "${prev}" in
|
||||
-e|--editor)
|
||||
_filedir
|
||||
|
||||
@ -30,14 +30,14 @@ gradle = "gradle"
|
||||
# The same policy is applied to the archive repo, if there is one.
|
||||
repo_maxage = 0
|
||||
|
||||
repo_url = "https://f-droid.org/repo"
|
||||
repo_name = "F-Droid"
|
||||
repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
|
||||
repo_name = "My First FDroid Repo Demo"
|
||||
repo_icon = "fdroid-icon.png"
|
||||
repo_description = """
|
||||
The official repository of the F-Droid client. Applications in this repository
|
||||
are either official binaries built by the original application developers, or
|
||||
are binaries built from source by the admin of f-droid.org using the tools on
|
||||
https://gitorious.org/f-droid.
|
||||
This is a repository of apps to be used with FDroid. Applications in this
|
||||
repository are either official binaries built by the original application
|
||||
developers, or are binaries built from source by the admin of f-droid.org
|
||||
using the tools on https://gitlab.com/u/fdroid.
|
||||
"""
|
||||
|
||||
# As above, but for the archive repo.
|
||||
@ -46,11 +46,11 @@ https://gitorious.org/f-droid.
|
||||
# repository, and no need to define the other archive_ values.
|
||||
archive_older = 3
|
||||
archive_url = "https://f-droid.org/archive"
|
||||
archive_name = "F-Droid Archive"
|
||||
archive_name = "My First FDroid Archive Demo"
|
||||
archive_icon = "fdroid-icon.png"
|
||||
archive_description = """
|
||||
The archive repository of the F-Droid client. This contains older versions
|
||||
of applications from the main repository.
|
||||
The repository of older versions of applications from the main demo
|
||||
repository.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@ -144,6 +144,23 @@ def test_sdk_exists(c):
|
||||
return True
|
||||
|
||||
|
||||
def test_build_tools_exists(c):
|
||||
if not test_sdk_exists(c):
|
||||
return False
|
||||
build_tools = os.path.join(c['sdk_path'], 'build-tools')
|
||||
versioned_build_tools = os.path.join(build_tools, c['build_tools'])
|
||||
if not os.path.isdir(versioned_build_tools):
|
||||
logging.critical('Android Build Tools path "'
|
||||
+ versioned_build_tools + '" does not exist!')
|
||||
return False
|
||||
if not os.path.exists(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'], 'aapt')):
|
||||
logging.critical('Android Build Tools "'
|
||||
+ versioned_build_tools
|
||||
+ '" does not contain "aapt"!')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def write_password_file(pwtype, password=None):
|
||||
'''
|
||||
writes out passwords to a protected file instead of passing passwords as
|
||||
@ -1514,7 +1531,7 @@ def SilentPopen(commands, cwd=None, shell=False):
|
||||
return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
|
||||
|
||||
|
||||
def FDroidPopen(commands, cwd=None, shell=False, output=True):
|
||||
def FDroidPopen(commands, cwd=None, shell=False, output=False):
|
||||
"""
|
||||
Run a command and capture the possibly huge output.
|
||||
|
||||
@ -1541,7 +1558,7 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True):
|
||||
while not stdout_reader.eof():
|
||||
while not stdout_queue.empty():
|
||||
line = stdout_queue.get()
|
||||
if output and options.verbose:
|
||||
if output or options.verbose:
|
||||
# Output directly to console
|
||||
sys.stdout.write(line)
|
||||
sys.stdout.flush()
|
||||
|
||||
@ -132,7 +132,12 @@ def main():
|
||||
# if neither --android-home nor the default sdk_path exist, prompt the user
|
||||
default_sdk_path = '/opt/android-sdk'
|
||||
while not options.no_prompt:
|
||||
s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ')
|
||||
try:
|
||||
s = raw_input('Enter the path to the Android SDK ('
|
||||
+ default_sdk_path + ') here:\n> ')
|
||||
except KeyboardInterrupt:
|
||||
print('')
|
||||
sys.exit(1)
|
||||
if re.match('^\s*$', s) is not None:
|
||||
test_config['sdk_path'] = default_sdk_path
|
||||
else:
|
||||
@ -155,18 +160,15 @@ def main():
|
||||
logging.info('Try running `fdroid init` in an empty directory.')
|
||||
sys.exit()
|
||||
|
||||
# now that we have a local config.py, read configuration...
|
||||
config = common.read_config(options)
|
||||
|
||||
# try to find a working aapt, in all the recent possible paths
|
||||
build_tools = os.path.join(config['sdk_path'], 'build-tools')
|
||||
build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
|
||||
aaptdirs = []
|
||||
aaptdirs.append(os.path.join(build_tools, config['build_tools']))
|
||||
aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
|
||||
aaptdirs.append(build_tools)
|
||||
for f in sorted(os.listdir(build_tools), reverse=True):
|
||||
for f in os.listdir(build_tools):
|
||||
if os.path.isdir(os.path.join(build_tools, f)):
|
||||
aaptdirs.append(os.path.join(build_tools, f))
|
||||
for d in aaptdirs:
|
||||
for d in sorted(aaptdirs, reverse=True):
|
||||
if os.path.isfile(os.path.join(d, 'aapt')):
|
||||
aapt = os.path.join(d, 'aapt')
|
||||
break
|
||||
@ -174,9 +176,15 @@ def main():
|
||||
dirname = os.path.basename(os.path.dirname(aapt))
|
||||
if dirname == 'build-tools':
|
||||
# this is the old layout, before versioned build-tools
|
||||
write_to_config('build_tools', '')
|
||||
test_config['build_tools'] = ''
|
||||
else:
|
||||
write_to_config('build_tools', dirname)
|
||||
test_config['build_tools'] = dirname
|
||||
write_to_config('build_tools', test_config['build_tools'])
|
||||
if not common.test_build_tools_exists(test_config):
|
||||
sys.exit(3)
|
||||
|
||||
# now that we have a local config.py, read configuration...
|
||||
config = common.read_config(options)
|
||||
|
||||
# track down where the Android NDK is
|
||||
ndk_path = '/opt/android-ndk'
|
||||
|
||||
@ -301,9 +301,6 @@ def resize_icon(iconpath, density):
|
||||
iconpath, oldsize, im.size))
|
||||
im.save(iconpath, "PNG")
|
||||
|
||||
else:
|
||||
logging.info("%s is small enough: %s" % im.size)
|
||||
|
||||
except Exception, e:
|
||||
logging.error("Failed resizing {0} - {1}".format(iconpath, e))
|
||||
|
||||
@ -366,8 +363,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
|
||||
thisinfo = apkcache[apkfilename]
|
||||
|
||||
else:
|
||||
|
||||
logging.info("Processing " + apkfilename)
|
||||
if options.verbose:
|
||||
logging.info("Processing " + apkfilename)
|
||||
thisinfo = {}
|
||||
thisinfo['apkname'] = apkfilename
|
||||
srcfilename = apkfilename[:-4] + "_src.tar.gz"
|
||||
@ -382,8 +379,15 @@ def scan_apks(apps, apkcache, repodir, knownapks):
|
||||
config['build_tools'], 'aapt'),
|
||||
'dump', 'badging', apkfile])
|
||||
if p.returncode != 0:
|
||||
logging.critical("Failed to get apk information")
|
||||
sys.exit(1)
|
||||
if options.delete_unknown:
|
||||
if os.path.exists(apkfile):
|
||||
logging.error("Failed to get apk information, deleting " + apkfile)
|
||||
os.remove(apkfile)
|
||||
else:
|
||||
logging.error("Could not find {0} to remove it".format(apkfile))
|
||||
else:
|
||||
logging.error("Failed to get apk information, skipping " + apkfile)
|
||||
continue
|
||||
for line in p.stdout.splitlines():
|
||||
if line.startswith("package:"):
|
||||
try:
|
||||
@ -415,7 +419,12 @@ def scan_apks(apps, apkcache, repodir, knownapks):
|
||||
path = match.group(2)
|
||||
thisinfo['icons_src'][density] = path
|
||||
elif line.startswith("sdkVersion:"):
|
||||
thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
|
||||
m = re.match(sdkversion_pat, line)
|
||||
if m is None:
|
||||
logging.error(line.replace('sdkVersion:', '')
|
||||
+ ' is not a valid minSdkVersion!')
|
||||
else:
|
||||
thisinfo['sdkversion'] = m.group(1)
|
||||
elif line.startswith("maxSdkVersion:"):
|
||||
thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
|
||||
elif line.startswith("native-code:"):
|
||||
@ -443,7 +452,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
|
||||
|
||||
# Check for debuggable apks...
|
||||
if common.isApkDebuggable(apkfile, config):
|
||||
logging.warn("{0} is debuggable... {1}".format(apkfile, line))
|
||||
logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile))
|
||||
|
||||
# Calculate the sha256...
|
||||
sha = hashlib.sha256()
|
||||
@ -804,8 +813,8 @@ def make_index(apps, apks, repodir, archive, categories):
|
||||
|
||||
if 'repo_keyalias' in config:
|
||||
|
||||
logging.info("Creating signed index with this key:")
|
||||
logging.info("SHA256: %s" % repo_pubkey_fingerprint)
|
||||
logging.info("Creating signed index with this key (SHA256):")
|
||||
logging.info("%s" % repo_pubkey_fingerprint)
|
||||
|
||||
# Create a jar of the index...
|
||||
p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
|
||||
@ -882,7 +891,7 @@ def main():
|
||||
|
||||
# Parse command line...
|
||||
parser = OptionParser()
|
||||
parser.add_option("-c", "--createmeta", action="store_true", default=False,
|
||||
parser.add_option("-c", "--create-metadata", action="store_true", default=False,
|
||||
help="Create skeleton metadata files that are missing")
|
||||
parser.add_option("--delete-unknown", action="store_true", default=False,
|
||||
help="Delete APKs without metadata from the repo")
|
||||
@ -957,7 +966,10 @@ def main():
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
if options.createmeta:
|
||||
if options.create_metadata:
|
||||
if 'name' not in apk:
|
||||
logging.error(apk['id'] + ' does not have a name! Skipping...')
|
||||
continue
|
||||
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
|
||||
f.write("License:Unknown\n")
|
||||
f.write("Web Site:\n")
|
||||
|
||||
@ -47,7 +47,7 @@ cd $WORKSPACE/fdroidserver/getsig
|
||||
#------------------------------------------------------------------------------#
|
||||
# run local tests
|
||||
cd $WORKSPACE/tests
|
||||
./run-tests
|
||||
./run-tests ~jenkins/
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
@ -67,12 +67,12 @@ python setup.py install
|
||||
|
||||
# run tests in new pip+virtualenv install
|
||||
. $WORKSPACE/env/bin/activate
|
||||
fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests
|
||||
fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
# run pyflakes
|
||||
pyflakes fdroid makebuildserver fdroidserver/*.py setup.py
|
||||
# run git pre-commit hook for pep8, pyflakes, etc
|
||||
sh hooks/pre-commit
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
|
||||
109
tests/run-tests
109
tests/run-tests
@ -3,13 +3,21 @@
|
||||
set -e
|
||||
set -x
|
||||
|
||||
echo_header() {
|
||||
echo "=============================================================================="
|
||||
echo $1
|
||||
}
|
||||
|
||||
copy_apks_into_repo() {
|
||||
for f in `ls -1 ../../*/bin/*.apk`; do
|
||||
for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned`; do
|
||||
name=$(basename $(dirname `dirname $f`))
|
||||
echo "name $name"
|
||||
apk=`aapt d badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
|
||||
echo "apk $apk"
|
||||
cp -f $f $1/repo/$apk
|
||||
apk=`aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
|
||||
test $f -nt repo/$apk && rm -f repo/$apk # delete existing if $f is newer
|
||||
if [ ! -e repo/$apk ] && [ ! -e archive/$apk ]; then
|
||||
echo "$f --> repo/$apk"
|
||||
ln $f $1/repo/$apk || \
|
||||
rsync -axv $f $1/repo/$apk # rsync if hard link is not possible
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
@ -29,6 +37,16 @@ create_test_file() {
|
||||
mktemp --tmpdir=$WORKSPACE/.testfiles
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
# "main"
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 '/path/to/folder/with/apks'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
APKDIR=$1
|
||||
|
||||
if [ -z $WORKSPACE ]; then
|
||||
WORKSPACE=`dirname $(pwd)`
|
||||
echo "Setting Workspace to $WORKSPACE"
|
||||
@ -40,14 +58,14 @@ if [ -z $fdroid ]; then
|
||||
fi
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "setup a new repo from scratch using ANDROID_HOME"
|
||||
echo_header "setup a new repo from scratch using ANDROID_HOME"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
cd $REPOROOT
|
||||
$fdroid init
|
||||
copy_apks_into_repo $REPOROOT
|
||||
$fdroid update -c
|
||||
$fdroid update
|
||||
$fdroid update --create-metadata
|
||||
grep -F '<application id=' repo/index.xml
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
@ -76,7 +94,33 @@ set -e
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "check that --android-home overrides ANDROID_HOME"
|
||||
echo_header "check that fake android home passes `fdroid init`"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
FAKE_ANDROID_HOME=`create_test_dir`
|
||||
create_fake_android_home $FAKE_ANDROID_HOME
|
||||
KEYSTORE=$REPOROOT/keystore.jks
|
||||
cd $REPOROOT
|
||||
$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "check that 'fdroid init' fails when build-tools cannot be found"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
FAKE_ANDROID_HOME=`create_test_dir`
|
||||
create_fake_android_home $FAKE_ANDROID_HOME
|
||||
rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt
|
||||
KEYSTORE=$REPOROOT/keystore.jks
|
||||
cd $REPOROOT
|
||||
set +e
|
||||
$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
|
||||
[ $? -eq 0 ] && exit 1
|
||||
set -e
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "check that --android-home overrides ANDROID_HOME"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
FAKE_ANDROID_HOME=`create_test_dir`
|
||||
@ -94,7 +138,12 @@ set -e
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "setup a new repo from scratch with keystore and android-home set on cmd line"
|
||||
# In this case, ANDROID_HOME is set to a fake, non-working version that will
|
||||
# be detected by fdroid as an Android SDK install. It should use the path set
|
||||
# by --android-home over the one in ANDROID_HOME, therefore if it uses the one
|
||||
# in ANDROID_HOME, it won't work because it is a fake one. Only
|
||||
# --android-home provides a working one.
|
||||
echo_header "setup a new repo from scratch with keystore and android-home set on cmd line"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
KEYSTORE=$REPOROOT/keystore.jks
|
||||
@ -104,30 +153,30 @@ STORED_ANDROID_HOME=$ANDROID_HOME
|
||||
unset ANDROID_HOME
|
||||
echo "ANDROID_HOME: $ANDROID_HOME"
|
||||
cd $REPOROOT
|
||||
$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME --no-prompt
|
||||
$fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-prompt
|
||||
test -e $KEYSTORE
|
||||
copy_apks_into_repo $REPOROOT
|
||||
$fdroid update -c
|
||||
$fdroid update
|
||||
$fdroid update --create-metadata
|
||||
grep -F '<application id=' repo/index.xml
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
export ANDROID_HOME=$STORED_ANDROID_HOME
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"
|
||||
echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
cd $REPOROOT
|
||||
mkdir repo
|
||||
copy_apks_into_repo $REPOROOT
|
||||
$fdroid init
|
||||
$fdroid update -c
|
||||
$fdroid update
|
||||
$fdroid update --create-metadata
|
||||
grep -F '<application id=' repo/index.xml
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "setup a new repo from scratch and generate a keystore"
|
||||
echo_header "setup a new repo from scratch and generate a keystore"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
KEYSTORE=$REPOROOT/keystore.jks
|
||||
@ -135,14 +184,34 @@ cd $REPOROOT
|
||||
$fdroid init --keystore $KEYSTORE
|
||||
test -e $KEYSTORE
|
||||
copy_apks_into_repo $REPOROOT
|
||||
$fdroid update -c
|
||||
$fdroid update
|
||||
$fdroid update --create-metadata
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
grep -F '<application id=' repo/index.xml
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo "setup a new repo from scratch with a HSM/smartcard"
|
||||
echo_header "setup a new repo from scratch, generate a keystore, then add APK and update"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
KEYSTORE=$REPOROOT/keystore.jks
|
||||
cd $REPOROOT
|
||||
$fdroid init --keystore $KEYSTORE
|
||||
test -e $KEYSTORE
|
||||
copy_apks_into_repo $REPOROOT
|
||||
$fdroid update --create-metadata
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
grep -F '<application id=' repo/index.xml
|
||||
cp $WORKSPACE/tests/urzip.apk $REPOROOT/
|
||||
$fdroid update --create-metadata
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
grep -F '<application id=' repo/index.xml
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "setup a new repo from scratch with a HSM/smartcard"
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
cd $REPOROOT
|
||||
|
||||
BIN
tests/urzip.apk
Normal file
BIN
tests/urzip.apk
Normal file
Binary file not shown.
Loading…
Reference in New Issue
Block a user