fdroidserver/tests/run-tests

#!/bin/bash

set -e # quit script on error

echo_header() {
    { echo -e "==============================================================================\n$1"; } 2>/dev/null
}

copy_apks_into_repo() {
    set +x
    for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned -e badsig -e badcert`; do
        name=$(basename $(dirname `dirname $f`))
        apk=`$aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
        test $f -nt repo/$apk && rm -f repo/$apk  # delete existing if $f is newer
        if [ ! -e repo/$apk ] && [ ! -e archive/$apk ]; then
            echo "$f --> repo/$apk"
            ln $f $1/repo/$apk || \
                rsync -axv $f $1/repo/$apk # rsync if hard link is not possible
        fi
    done
    set -x
}

# keep this as an old version to test the automatic parsing of build-tools
# verion numbers in `fdroid init`
create_fake_android_home() {
    mkdir $1/tools
    mkdir $1/platform-tools
    mkdir $1/build-tools
    mkdir $1/build-tools/19.0.2
    touch $1/build-tools/19.0.2/aapt
}

create_test_dir() {
    test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
    mktemp -d $WORKSPACE/.testfiles/run-tests.XXXX
}

create_test_file() {
    test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
    TMPDIR=$WORKSPACE/.testfiles  mktemp
}

#------------------------------------------------------------------------------#
# "main"

if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
    set +x
    echo "Usage: $0 '/path/to/folder/with/apks'"
    exit 1
fi

if [ -z "$ANDROID_HOME" ]; then
    echo "ANDROID_HOME must be set with the path to the Android SDK, i.e.: "
    echo "  export ANDROID_HOME=/opt/android-sdk"
    exit 1
fi

if [ -d tests ]; then
    cd tests
fi

if [ -z "$1" ]; then
    APKDIR=`pwd`
else
    APKDIR=$1
fi

if [ -z $WORKSPACE ]; then
    WORKSPACE=`dirname $(pwd)`
    echo "Setting Workspace to $WORKSPACE"
fi

# allow the location of the script to be overridden
if [ -z $fdroid ]; then
    fdroid="$WORKSPACE/fdroid"
fi

# allow the location of aapt to be overridden
if [ -z $aapt ]; then
    aapt=`ls -1 $ANDROID_HOME/build-tools/*/aapt | sort | tail -1`
fi

# allow the location of python to be overridden
if [ -z $python ]; then
    python=python3
fi

set -x # show each command as it is executed

#------------------------------------------------------------------------------#
echo_header "run commit hooks"

cd $WORKSPACE
./hooks/pre-commit


#------------------------------------------------------------------------------#
echo_header "test python getsig replacement"

cd $WORKSPACE/tests/getsig
./make.sh

cd $WORKSPACE/tests
for testcase in $WORKSPACE/tests/*.TestCase; do
    $testcase
done


#------------------------------------------------------------------------------#
echo_header "print fdroid version"

$fdroid --version


#------------------------------------------------------------------------------#
echo_header 'run process when building and signing are on separate machines'

REPOROOT=`create_test_dir`
cd $REPOROOT
cp $WORKSPACE/tests/keystore.jks $REPOROOT/
$fdroid init --keystore keystore.jks --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo "accepted_formats = ['txt', 'yml']" >> config.py
echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py
test -d archive || mkdir archive
test -d metadata || mkdir metadata
cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/
test -d repo || mkdir repo
test -d unsigned || mkdir unsigned
cp $WORKSPACE/tests/urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk
$fdroid publish --verbose
$fdroid update --verbose --nosign
$fdroid signindex --verbose
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
test -L urzip.apk
grep -F '<application id=' repo/index.xml > /dev/null


#------------------------------------------------------------------------------#
echo_header "test UTF-8 metadata"

REPOROOT=`create_test_dir`
cd $REPOROOT

$fdroid init
sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您的设备上的应用的,' config.py
echo "mirrors = ('https://foo.bar/fdroid', 'http://secret.onion/fdroid')" >> config.py
mkdir metadata
cp $WORKSPACE/tests/urzip.apk repo/
cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/

$fdroid readmeta
$fdroid update


#------------------------------------------------------------------------------#
echo_header 'run `fdroid build` in fresh git checkout from import.TestCase'

cd $WORKSPACE/tests/tmp/importer
git remote update -p
git clean -fdx
# stick with known working commit, in case future commits break things for this code
git reset --hard cecf00c08aec56ae7a5eba444150c4d1ae868814
if [ -d $ANDROID_HOME/platforms/android-23 ]; then
    echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py
    echo "force_build_tools = True" >> config.py
    $fdroid build --verbose org.fdroid.ci.test.app:300
else
    echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!'
fi


#------------------------------------------------------------------------------#
echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"

REPOROOT=`create_test_dir`
GNUPGHOME=$REPOROOT/gnupghome
KEYSTORE=$WORKSPACE/tests/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
chmod 0700 $GNUPGHOME
echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
echo "install_list = 'org.adaway'" >> config.py
echo "uninstall_list = ('com.android.vending', 'com.facebook.orca',)" >> config.py
echo "gpghome = '$GNUPGHOME'" >> config.py
echo "gpgkey = 'CE71F7FB'" >> config.py
echo "mirrors = ('http://foobarfoobarfoobar.onion/fdroid','https://foo.bar/fdroid',)" >> config.py
$fdroid update --verbose --pretty
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null
grep -F '<install packageName=' repo/index.xml > /dev/null
grep -F '<uninstall packageName=' repo/index.xml > /dev/null
$fdroid gpgsign --verbose
$fdroid gpgsign --verbose
test -e repo/obb.mainpatch.current_1619.apk.asc
test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
! test -e repo/obb.mainpatch.current_1619.apk.asc.asc
! test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc.asc
! test -e repo/index.xml.asc

# we can't easily reproduce the timestamps for things, so just hardcode them
sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml
diff $WORKSPACE/tests/repo/index.xml repo/index.xml

#------------------------------------------------------------------------------#
echo_header "test metadata checks"

REPOROOT=`create_test_dir`
cd $REPOROOT

touch config.py
mkdir repo
cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/

set +e
$fdroid build
if [ $? -eq 0 ]; then
    echo "This should have failed because there is no metadata!"
    exit 1
else
    echo "testing metadata checks passed"
fi
set -e

mkdir $REPOROOT/metadata/
cp $WORKSPACE/tests/metadata/org.smssecure.smssecure.txt $REPOROOT/metadata/
$fdroid readmeta

# now make a fake duplicate
touch $REPOROOT/metadata/org.smssecure.smssecure.yml

set +e
$fdroid readmeta
if [ $? -eq 0 ]; then
    echo "This should have failed because there is a duplicate metadata file!"
    exit 1
else
    echo "testing duplicate metadata checks passed"
fi
set -e


#------------------------------------------------------------------------------#
echo_header "ensure commands that don't need the JDK work without a JDK configed"

REPOROOT=`create_test_dir`
cd $REPOROOT
mkdir repo
mkdir metadata
echo "License:GPL" >> metadata/fake.txt
echo "Summary:Yup still fake" >> metadata/fake.txt
echo "Categories:Internet" >> metadata/fake.txt
echo "Description:" >> metadata/fake.txt
echo "this is fake" >> metadata/fake.txt
echo "." >> metadata/fake.txt

# fake that no JDKs are available
echo 'java_paths = {}' > config.py

LOCAL_COPY_DIR=`create_test_dir`/fdroid
mkdir -p $LOCAL_COPY_DIR/repo
echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py

$fdroid checkupdates
$fdroid gpgsign
$fdroid lint
$fdroid readmeta
$fdroid rewritemeta fake
$fdroid server update
$fdroid scanner

# run these to get their output, but the are not setup, so don't fail
$fdroid build || true
$fdroid import || true
$fdroid install || true


#------------------------------------------------------------------------------#
echo_header "create a source tarball and use that to build a repo"

cd $WORKSPACE
$python setup.py sdist

REPOROOT=`create_test_dir`
cd $REPOROOT
tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1`
cd $REPOROOT
./fdroidserver-*/fdroid init
copy_apks_into_repo $REPOROOT
./fdroidserver-*/fdroid update --create-metadata --verbose


#------------------------------------------------------------------------------#
echo_header "test config checks of local_copy_dir"

REPOROOT=`create_test_dir`
cd $REPOROOT
$fdroid init
$fdroid update --create-metadata --verbose
$fdroid readmeta
$fdroid server update --local-copy-dir=/tmp/fdroid

# now test the errors work
set +e
$fdroid server update --local-copy-dir=thisisnotanabsolutepath
if [ $? -eq 0 ]; then
    echo "This should have failed because thisisnotanabsolutepath is not an absolute path!"
    exit 1
else
    echo "testing absolute path checker passed"
fi
$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf
if [ $? -eq 0 ]; then
    echo "This should have failed because the path does not end with 'fdroid'!"
    exit 1
else
    echo "testing dirname exists checker passed"
fi
$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf/fdroid
if [ $? -eq 0 ]; then
    echo "This should have failed because the dirname path does not exist!"
    exit 1
else
    echo "testing dirname exists checker passed"
fi
set -e


#------------------------------------------------------------------------------#
echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync"

REPOROOT=`create_test_dir`
cd $REPOROOT
$fdroid init
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata --verbose
$fdroid readmeta
grep -F '<application id=' repo/index.xml > /dev/null

LOCALCOPYDIR=`create_test_dir`/fdroid
$fdroid server update --local-copy-dir=$LOCALCOPYDIR
NEWREPOROOT=`create_test_dir`
cd $NEWREPOROOT
$fdroid init
$fdroid server update --local-copy-dir=$LOCALCOPYDIR --sync-from-local-copy-dir


#------------------------------------------------------------------------------#
# check that --android-home fails when dir does not exist or is not a dir

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
set +e
$fdroid init --keystore $KEYSTORE --android-home /opt/fakeandroidhome
if [ $? -eq 0 ]; then
    echo "This should have failed because /opt/fakeandroidhome does not exist!"
    exit 1
else
    echo "testing android-home path checker passed"
fi
TESTFILE=`create_test_file`
$fdroid init --keystore $KEYSTORE --android-home $TESTFILE
if [ $? -eq 0 ]; then
    echo "This should have failed because $TESTFILE is a file not a dir!"
    exit 1
else
    echo "testing android-home not-dir checker passed"
fi
set -e


#------------------------------------------------------------------------------#
echo_header "check that fake android home passes 'fdroid init'"

REPOROOT=`create_test_dir`
FAKE_ANDROID_HOME=`create_test_dir`
create_fake_android_home $FAKE_ANDROID_HOME
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME


#------------------------------------------------------------------------------#
echo_header "check that 'fdroid init' fails when build-tools cannot be found"

if [ -e /usr/bin/aapt ]; then
    echo "/usr/bin/aapt exists, not running test"
else
    REPOROOT=`create_test_dir`
    FAKE_ANDROID_HOME=`create_test_dir`
    create_fake_android_home $FAKE_ANDROID_HOME
    rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt
    KEYSTORE=$REPOROOT/keystore.jks
    cd $REPOROOT
    set +e
    $fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
    [ $? -eq 0 ] && exit 1
    set -e
fi


#------------------------------------------------------------------------------#
echo_header "check that --android-home overrides ANDROID_HOME"

REPOROOT=`create_test_dir`
FAKE_ANDROID_HOME=`create_test_dir`
create_fake_android_home $FAKE_ANDROID_HOME
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
set +e
grep $FAKE_ANDROID_HOME $REPOROOT/config.py
if [ $? -ne 0 ]; then
    echo "the value set in --android-home '$FAKE_ANDROID_HOME' should override ANDROID_HOME '$ANDROID_HOME'"
    exit 1
fi
set -e


#------------------------------------------------------------------------------#
# In this case, ANDROID_HOME is set to a fake, non-working version that will
# be detected by fdroid as an Android SDK install.  It should use the path set
# by --android-home over the one in ANDROID_HOME, therefore if it uses the one
# in ANDROID_HOME, it won't work because it is a fake one.  Only
# --android-home provides a working one.
echo_header "setup a new repo from scratch with keystore and android-home set on cmd line"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
FAKE_ANDROID_HOME=`create_test_dir`
create_fake_android_home $FAKE_ANDROID_HOME
STORED_ANDROID_HOME=$ANDROID_HOME
unset ANDROID_HOME
echo "ANDROID_HOME: $ANDROID_HOME"
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-prompt
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata --verbose
$fdroid readmeta
grep -F '<application id=' repo/index.xml > /dev/null
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
export ANDROID_HOME=$STORED_ANDROID_HOME


#------------------------------------------------------------------------------#
echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"

REPOROOT=`create_test_dir`
cd $REPOROOT
mkdir repo
copy_apks_into_repo $REPOROOT
$fdroid init
$fdroid update --create-metadata --verbose
$fdroid readmeta
grep -F '<application id=' repo/index.xml > /dev/null


#------------------------------------------------------------------------------#
echo_header "setup a new repo from scratch and generate a keystore"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata --verbose
$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null


#------------------------------------------------------------------------------#
echo_header "setup a new repo manually and generate a keystore"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
touch config.py
cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
! test -e $KEYSTORE
set +e
$fdroid update
if [ $? -eq 0 ]; then
    echo "This should have failed because this repo has no keystore!"
    exit 1
else
    echo '`fdroid update` prompted to add keystore'
fi
set -e
$fdroid update --create-key
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata --verbose
$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null


#------------------------------------------------------------------------------#
echo_header "setup a new repo from scratch, generate a keystore, then add APK and update"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata --verbose
$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null
test -e $REPOROOT/repo/info.guardianproject.urzip_100.apk || \
    cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
$fdroid update --create-metadata --verbose
$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null


#------------------------------------------------------------------------------#
echo_header "setup a new repo from scratch with a HSM/smartcard"
REPOROOT=`create_test_dir`
cd $REPOROOT
$fdroid init --keystore NONE
test -e opensc-fdroid.cfg
test ! -e NONE


#------------------------------------------------------------------------------#
echo_header "setup a new repo with no keystore, add APK, and update"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
touch config.py
touch fdroid-icon.png
mkdir repo
cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
set +e
$fdroid update --create-metadata --verbose
if [ $? -eq 0 ]; then
    echo "This should have failed because this repo has no keystore!"
    exit 1
else
    echo '`fdroid update` prompted to add keystore'
fi
set -e

# now set up fake, non-working keystore setup
touch $KEYSTORE
echo "keystore = \"$KEYSTORE\"" >> config.py
echo 'repo_keyalias = "foo"' >> config.py
echo 'keystorepass = "foo"' >> config.py
echo 'keypass = "foo"' >> config.py
set +e
$fdroid update --create-metadata --verbose
if [ $? -eq 0 ]; then
    echo "This should have failed because this repo has a bad/fake keystore!"
    exit 1
else
    echo '`fdroid update` prompted to add keystore'
fi
set -e


#------------------------------------------------------------------------------#
echo_header "copy tests/repo, update with binary transparency log"

REPOROOT=`create_test_dir`
GNUPGHOME=$REPOROOT/gnupghome
KEYSTORE=$WORKSPACE/tests/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py
echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
$fdroid update --verbose --pretty
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null
cd binary_transparency
[ `git rev-list --count HEAD` == "2" ]


#------------------------------------------------------------------------------#
echo_header "setup a new repo with keystore with APK, update, then without key"

REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE
test -e $KEYSTORE
cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
$fdroid update --create-metadata --verbose
$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null

# now set fake repo_keyalias
sed -i.tmp 's,^ *repo_keyalias.*,repo_keyalias = "fake",' $REPOROOT/config.py
set +e
$fdroid update
if [ $? -eq 0 ]; then
    echo "This should have failed because this repo has a bad repo_keyalias!"
    exit 1
else
    echo '`fdroid update` prompted to add keystore'
fi
set -e

# try creating a new keystore, but fail because the old one is there
test -e $KEYSTORE
set +e
$fdroid update --create-key
if [ $? -eq 0 ]; then
    echo "This should have failed because a keystore is already there!"
    exit 1
else
    echo '`fdroid update` complained about existing keystore'
fi
set -e

# now actually create the key with the existing settings
rm -f $KEYSTORE
! test -e $KEYSTORE
$fdroid update --create-key
test -e $KEYSTORE


#------------------------------------------------------------------------------#

# remove this to prevent git conflicts and complaining
rm -rf $WORKSPACE/fdroidserver.egg-info/

echo SUCCESS