Merge branch 'test-updates-and-related-bug-fixes' into 'master'

Test updates and related bug fixes

I just set up some big tests of generating repos based on feeding as many random APKs into `fdroid update` as possible.  On our jenkins server, the tests copy all of the APKs that the jenkins server has generated and builds a repo from them.  This process caught lots of little glitches in the whole process.  While these little glitches are usually caused by problematic APKs, `fdroid update` should handle them gracefully.  Hopefully this set of fixes accomplishes that.

completion/bash-completion

@@ -128,8 +128,8 @@ __complete_install() {
 
 __complete_update() {
 	opts="-h -c -v -q -b -i -I -e -w"
-	lopts="--help --createmeta --verbose --quiet --buildreport --interactive
+	lopts="--help --create-metadata --verbose --quiet --buildreport
- --icons --editor --wiki --pretty --clean --delete-unknown"
+ --interactive --icons --editor --wiki --pretty --clean --delete-unknown"
 	case "${prev}" in
 		-e|--editor)
 			_filedir

examples/config.py

@@ -30,14 +30,14 @@ gradle = "gradle"
 # The same policy is applied to the archive repo, if there is one.
 repo_maxage = 0
 
-repo_url = "https://f-droid.org/repo"
+repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
-repo_name = "F-Droid"
+repo_name = "My First FDroid Repo Demo"
 repo_icon = "fdroid-icon.png"
 repo_description = """
-The official repository of the F-Droid client. Applications in this repository
+This is a repository of apps to be used with FDroid. Applications in this
-are either official binaries built by the original application developers, or
+repository are either official binaries built by the original application
-are binaries built from source by the admin of f-droid.org using the tools on
+developers, or are binaries built from source by the admin of f-droid.org
-https://gitorious.org/f-droid.
+using the tools on https://gitlab.com/u/fdroid.
 """
 
 # As above, but for the archive repo.
@@ -46,11 +46,11 @@ https://gitorious.org/f-droid.
 # repository, and no need to define the other archive_ values.
 archive_older = 3
 archive_url = "https://f-droid.org/archive"
-archive_name = "F-Droid Archive"
+archive_name = "My First FDroid Archive Demo"
 archive_icon = "fdroid-icon.png"
 archive_description = """
-The archive repository of the F-Droid client. This contains older versions
+The repository of older versions of applications from the main demo
-of applications from the main repository.
+repository.
 """
 
 

fdroidserver/common.py

@@ -144,6 +144,23 @@ def test_sdk_exists(c):
     return True
 
 
+def test_build_tools_exists(c):
+    if not test_sdk_exists(c):
+        return False
+    build_tools = os.path.join(c['sdk_path'], 'build-tools')
+    versioned_build_tools = os.path.join(build_tools, c['build_tools'])
+    if not os.path.isdir(versioned_build_tools):
+        logging.critical('Android Build Tools path "'
+                         + versioned_build_tools + '" does not exist!')
+        return False
+    if not os.path.exists(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'], 'aapt')):
+        logging.critical('Android Build Tools "'
+                         + versioned_build_tools
+                         + '" does not contain "aapt"!')
+        return False
+    return True
+
+
 def write_password_file(pwtype, password=None):
     '''
     writes out passwords to a protected file instead of passing passwords as
@@ -1514,7 +1531,7 @@ def SilentPopen(commands, cwd=None, shell=False):
     return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
 
 
-def FDroidPopen(commands, cwd=None, shell=False, output=True):
+def FDroidPopen(commands, cwd=None, shell=False, output=False):
     """
     Run a command and capture the possibly huge output.
 
@@ -1541,7 +1558,7 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True):
     while not stdout_reader.eof():
         while not stdout_queue.empty():
             line = stdout_queue.get()
-            if output and options.verbose:
+            if output or options.verbose:
                 # Output directly to console
                 sys.stdout.write(line)
                 sys.stdout.flush()

fdroidserver/init.py

@@ -132,7 +132,12 @@ def main():
         # if neither --android-home nor the default sdk_path exist, prompt the user
         default_sdk_path = '/opt/android-sdk'
         while not options.no_prompt:
-            s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ')
+            try:
+                s = raw_input('Enter the path to the Android SDK ('
+                              + default_sdk_path + ') here:\n> ')
+            except KeyboardInterrupt:
+                print('')
+                sys.exit(1)
             if re.match('^\s*$', s) is not None:
                 test_config['sdk_path'] = default_sdk_path
             else:
@@ -155,18 +160,15 @@ def main():
         logging.info('Try running `fdroid init` in an empty directory.')
         sys.exit()
 
-    # now that we have a local config.py, read configuration...
-    config = common.read_config(options)
-
     # try to find a working aapt, in all the recent possible paths
-    build_tools = os.path.join(config['sdk_path'], 'build-tools')
+    build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
     aaptdirs = []
-    aaptdirs.append(os.path.join(build_tools, config['build_tools']))
+    aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
     aaptdirs.append(build_tools)
-    for f in sorted(os.listdir(build_tools), reverse=True):
+    for f in os.listdir(build_tools):
         if os.path.isdir(os.path.join(build_tools, f)):
             aaptdirs.append(os.path.join(build_tools, f))
-    for d in aaptdirs:
+    for d in sorted(aaptdirs, reverse=True):
         if os.path.isfile(os.path.join(d, 'aapt')):
             aapt = os.path.join(d, 'aapt')
             break
@@ -174,9 +176,15 @@ def main():
         dirname = os.path.basename(os.path.dirname(aapt))
         if dirname == 'build-tools':
             # this is the old layout, before versioned build-tools
-            write_to_config('build_tools', '')
+            test_config['build_tools'] = ''
         else:
-            write_to_config('build_tools', dirname)
+            test_config['build_tools'] = dirname
+        write_to_config('build_tools', test_config['build_tools'])
+    if not common.test_build_tools_exists(test_config):
+        sys.exit(3)
+
+    # now that we have a local config.py, read configuration...
+    config = common.read_config(options)
 
     # track down where the Android NDK is
     ndk_path = '/opt/android-ndk'

fdroidserver/update.py

@@ -301,9 +301,6 @@ def resize_icon(iconpath, density):
                 iconpath, oldsize, im.size))
             im.save(iconpath, "PNG")
 
-        else:
-            logging.info("%s is small enough: %s" % im.size)
-
     except Exception, e:
         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 
@@ -366,8 +363,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
             thisinfo = apkcache[apkfilename]
 
         else:
-
+            if options.verbose:
-            logging.info("Processing " + apkfilename)
+                logging.info("Processing " + apkfilename)
             thisinfo = {}
             thisinfo['apkname'] = apkfilename
             srcfilename = apkfilename[:-4] + "_src.tar.gz"
@@ -382,8 +379,15 @@ def scan_apks(apps, apkcache, repodir, knownapks):
                                           config['build_tools'], 'aapt'),
                              'dump', 'badging', apkfile])
             if p.returncode != 0:
-                logging.critical("Failed to get apk information")
+                if options.delete_unknown:
-                sys.exit(1)
+                    if os.path.exists(apkfile):
+                        logging.error("Failed to get apk information, deleting " + apkfile)
+                        os.remove(apkfile)
+                    else:
+                        logging.error("Could not find {0} to remove it".format(apkfile))
+                else:
+                    logging.error("Failed to get apk information, skipping " + apkfile)
+                continue
             for line in p.stdout.splitlines():
                 if line.startswith("package:"):
                     try:
@@ -415,7 +419,12 @@ def scan_apks(apps, apkcache, repodir, knownapks):
                         path = match.group(2)
                         thisinfo['icons_src'][density] = path
                 elif line.startswith("sdkVersion:"):
-                    thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
+                    m = re.match(sdkversion_pat, line)
+                    if m is None:
+                        logging.error(line.replace('sdkVersion:', '')
+                                      + ' is not a valid minSdkVersion!')
+                    else:
+                        thisinfo['sdkversion'] = m.group(1)
                 elif line.startswith("maxSdkVersion:"):
                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
                 elif line.startswith("native-code:"):
@@ -443,7 +452,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
 
             # Check for debuggable apks...
             if common.isApkDebuggable(apkfile, config):
-                logging.warn("{0} is debuggable... {1}".format(apkfile, line))
+                logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile))
 
             # Calculate the sha256...
             sha = hashlib.sha256()
@@ -804,8 +813,8 @@ def make_index(apps, apks, repodir, archive, categories):
 
     if 'repo_keyalias' in config:
 
-        logging.info("Creating signed index with this key:")
+        logging.info("Creating signed index with this key (SHA256):")
-        logging.info("SHA256: %s" % repo_pubkey_fingerprint)
+        logging.info("%s" % repo_pubkey_fingerprint)
 
         # Create a jar of the index...
         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
@@ -882,7 +891,7 @@ def main():
 
     # Parse command line...
     parser = OptionParser()
-    parser.add_option("-c", "--createmeta", action="store_true", default=False,
+    parser.add_option("-c", "--create-metadata", action="store_true", default=False,
                       help="Create skeleton metadata files that are missing")
     parser.add_option("--delete-unknown", action="store_true", default=False,
                       help="Delete APKs without metadata from the repo")
@@ -957,7 +966,10 @@ def main():
                 found = True
                 break
         if not found:
-            if options.createmeta:
+            if options.create_metadata:
+                if 'name' not in apk:
+                    logging.error(apk['id'] + ' does not have a name! Skipping...')
+                    continue
                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
                 f.write("License:Unknown\n")
                 f.write("Web Site:\n")

jenkins-build

@@ -47,7 +47,7 @@ cd $WORKSPACE/fdroidserver/getsig
 #------------------------------------------------------------------------------#
 # run local tests
 cd $WORKSPACE/tests
-./run-tests
+./run-tests ~jenkins/
 
 
 #------------------------------------------------------------------------------#
@@ -67,12 +67,12 @@ python setup.py install
 
 # run tests in new pip+virtualenv install
 . $WORKSPACE/env/bin/activate
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
 
 
 #------------------------------------------------------------------------------#
-# run pyflakes
+# run git pre-commit hook for pep8, pyflakes, etc
-pyflakes fdroid makebuildserver fdroidserver/*.py setup.py
+sh hooks/pre-commit
 
 
 #------------------------------------------------------------------------------#

tests/run-tests

@@ -3,13 +3,21 @@
 set -e
 set -x
 
+echo_header() {
+    echo "=============================================================================="
+    echo $1
+}
+
 copy_apks_into_repo() {
-    for f in `ls -1 ../../*/bin/*.apk`; do
+    for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned`; do
         name=$(basename $(dirname `dirname $f`))
-        echo "name $name"
+        apk=`aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
-        apk=`aapt d badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
+        test $f -nt repo/$apk && rm -f repo/$apk  # delete existing if $f is newer
-        echo "apk $apk"
+        if [ ! -e repo/$apk ] && [ ! -e archive/$apk ]; then
-        cp -f $f $1/repo/$apk
+            echo "$f --> repo/$apk"
+            ln $f $1/repo/$apk || \
+                rsync -axv $f $1/repo/$apk # rsync if hard link is not possible
+        fi
     done
 }
 
@@ -29,6 +37,16 @@ create_test_file() {
     mktemp --tmpdir=$WORKSPACE/.testfiles
 }
 
+#------------------------------------------------------------------------------#
+# "main"
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 '/path/to/folder/with/apks'"
+    exit 1
+fi
+
+APKDIR=$1
+
 if [ -z $WORKSPACE ]; then
     WORKSPACE=`dirname $(pwd)`
     echo "Setting Workspace to $WORKSPACE"
@@ -40,14 +58,14 @@ if [ -z $fdroid ]; then
 fi
 
 #------------------------------------------------------------------------------#
-echo "setup a new repo from scratch using ANDROID_HOME"
+echo_header "setup a new repo from scratch using ANDROID_HOME"
 
 REPOROOT=`create_test_dir`
 cd $REPOROOT
 $fdroid init
 copy_apks_into_repo $REPOROOT
-$fdroid update -c
+$fdroid update --create-metadata
-$fdroid update
+grep -F '<application id=' repo/index.xml
 
 
 #------------------------------------------------------------------------------#
@@ -76,7 +94,33 @@ set -e
 
 
 #------------------------------------------------------------------------------#
-echo "check that --android-home overrides ANDROID_HOME"
+echo_header "check that fake android home passes `fdroid init`"
+
+REPOROOT=`create_test_dir`
+FAKE_ANDROID_HOME=`create_test_dir`
+create_fake_android_home $FAKE_ANDROID_HOME
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
+
+
+#------------------------------------------------------------------------------#
+echo_header "check that 'fdroid init' fails when build-tools cannot be found"
+
+REPOROOT=`create_test_dir`
+FAKE_ANDROID_HOME=`create_test_dir`
+create_fake_android_home $FAKE_ANDROID_HOME
+rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+set +e
+$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
+[ $? -eq 0 ] && exit 1
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo_header "check that --android-home overrides ANDROID_HOME"
 
 REPOROOT=`create_test_dir`
 FAKE_ANDROID_HOME=`create_test_dir`
@@ -94,7 +138,12 @@ set -e
 
 
 #------------------------------------------------------------------------------#
-echo "setup a new repo from scratch with keystore and android-home set on cmd line"
+# In this case, ANDROID_HOME is set to a fake, non-working version that will
+# be detected by fdroid as an Android SDK install.  It should use the path set
+# by --android-home over the one in ANDROID_HOME, therefore if it uses the one
+# in ANDROID_HOME, it won't work because it is a fake one.  Only
+# --android-home provides a working one.
+echo_header "setup a new repo from scratch with keystore and android-home set on cmd line"
 
 REPOROOT=`create_test_dir`
 KEYSTORE=$REPOROOT/keystore.jks
@@ -104,30 +153,30 @@ STORED_ANDROID_HOME=$ANDROID_HOME
 unset ANDROID_HOME
 echo "ANDROID_HOME: $ANDROID_HOME"
 cd $REPOROOT
-$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME --no-prompt
+$fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-prompt
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update -c
+$fdroid update --create-metadata
-$fdroid update
+grep -F '<application id=' repo/index.xml
 test -e repo/index.xml
 test -e repo/index.jar
 export ANDROID_HOME=$STORED_ANDROID_HOME
 
 
 #------------------------------------------------------------------------------#
-echo "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"
+echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first"
 
 REPOROOT=`create_test_dir`
 cd $REPOROOT
 mkdir repo
 copy_apks_into_repo $REPOROOT
 $fdroid init
-$fdroid update -c
+$fdroid update --create-metadata
-$fdroid update
+grep -F '<application id=' repo/index.xml
 
 
 #------------------------------------------------------------------------------#
-echo "setup a new repo from scratch and generate a keystore"
+echo_header "setup a new repo from scratch and generate a keystore"
 
 REPOROOT=`create_test_dir`
 KEYSTORE=$REPOROOT/keystore.jks
@@ -135,14 +184,34 @@ cd $REPOROOT
 $fdroid init --keystore $KEYSTORE
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
-$fdroid update -c
+$fdroid update --create-metadata
-$fdroid update
 test -e repo/index.xml
 test -e repo/index.jar
+grep -F '<application id=' repo/index.xml
 
 
 #------------------------------------------------------------------------------#
-echo "setup a new repo from scratch with a HSM/smartcard"
+echo_header "setup a new repo from scratch, generate a keystore, then add APK and update"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE
+test -e $KEYSTORE
+copy_apks_into_repo $REPOROOT
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo from scratch with a HSM/smartcard"
 
 REPOROOT=`create_test_dir`
 cd $REPOROOT