Merge branch 'feature/dscanner' into master

dscanner - drozer scanner work.

closes !187

README.md

@@ -76,3 +76,19 @@ Then here's how to install:
 	source env/bin/activate
 	pip3 install -e .
 	python3 setup.py install
+
+
+### Drozer Scanner
+
+There is a new feature under development that can scan any APK in a
+repo, or any build, using Drozer.  Drozer is a dynamic exploit
+scanner, it runs an app in the emulator and runs known exploits on it.
+
+This setup requires specific versions of two Python modules:
+_docker-py_ 1.9.0 and _requests_ older than 2.11.  Other versions
+might cause the docker-py connection to break with the containers.
+Newer versions of docker-py might have this fixed already.
+
+For Debian based distributions:
+
+	apt-get install libffi-dev libssl-dev python-docker

docker/Dockerfile

@@ -0,0 +1,180 @@
+# This image is intended to be used with fdroidserver for the purpose
+# of dynamic scanning of pre-built APKs during the fdroid build process.
+
+# Start with ubuntu 12.04 (i386).
+FROM ubuntu:14.04
+MAINTAINER fdroid.dscanner <fdroid.dscanner@gmail.com>
+
+ENV DROZER_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer_2.3.4.deb
+ENV DROZER_DEB drozer_2.3.4.deb
+
+ENV AGENT_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer-agent-2.3.4.apk
+ENV AGENT_APK drozer-agent-2.3.4.apk
+
+# Specially for SSH access and port redirection
+ENV ROOTPASSWORD android
+
+# Expose ADB, ADB control and VNC ports
+EXPOSE 22
+EXPOSE 5037
+EXPOSE 5554
+EXPOSE 5555
+EXPOSE 5900
+EXPOSE 5901
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
+RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections
+
+# Update packages
+RUN apt-get -y update
+
+# Drozer packages
+RUN apt-get install wget python2.7 python-dev python2.7-dev python-openssl python-twisted python-protobuf bash-completion -y
+
+# First, install add-apt-repository, sshd and bzip2
+RUN apt-get -y install python-software-properties bzip2 ssh net-tools
+
+# ubuntu 14.04 needs this too
+RUN apt-get -y install software-properties-common
+
+# Add oracle-jdk7 to repositories
+RUN add-apt-repository ppa:webupd8team/java
+
+# Make sure the package repository is up to date
+RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
+
+# Update apt
+RUN apt-get update
+
+# Add drozer
+RUN useradd -ms /bin/bash drozer
+
+# Install oracle-jdk7
+RUN apt-get -y install oracle-java7-installer
+
+# Install android sdk
+RUN wget http://dl.google.com/android/android-sdk_r23-linux.tgz
+RUN tar -xvzf android-sdk_r23-linux.tgz
+RUN mv -v android-sdk-linux /usr/local/android-sdk
+
+# Install apache ant
+RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.8.4-bin.tar.gz
+RUN tar -xvzf apache-ant-1.8.4-bin.tar.gz
+RUN mv -v apache-ant-1.8.4 /usr/local/apache-ant
+
+# Add android tools and platform tools to PATH
+ENV ANDROID_HOME /usr/local/android-sdk
+ENV PATH $PATH:$ANDROID_HOME/tools
+ENV PATH $PATH:$ANDROID_HOME/platform-tools
+
+# Add ant to PATH
+ENV ANT_HOME /usr/local/apache-ant
+ENV PATH $PATH:$ANT_HOME/bin
+
+# Export JAVA_HOME variable
+ENV JAVA_HOME /usr/lib/jvm/java-7-oracle
+
+# Remove compressed files.
+RUN cd /; rm android-sdk_r23-linux.tgz && rm apache-ant-1.8.4-bin.tar.gz
+
+# Some preparation before update
+RUN chown -R root:root /usr/local/android-sdk/
+
+# Install latest android tools and system images
+RUN echo "y" | android update sdk --filter platform-tool --no-ui --force
+RUN echo "y" | android update sdk --filter platform --no-ui --force
+RUN echo "y" | android update sdk --filter build-tools-22.0.1 --no-ui -a
+RUN echo "y" | android update sdk --filter sys-img-x86-android-19 --no-ui -a
+#RUN echo "y" | android update sdk --filter sys-img-x86-android-21 --no-ui -a
+#RUN echo "y" | android update sdk --filter sys-img-x86-android-22 --no-ui -a
+RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-19 --no-ui -a
+#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-21 --no-ui -a
+#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-22 --no-ui -a
+
+# Update ADB
+RUN echo "y" | android update adb
+
+# Create fake keymap file
+RUN mkdir /usr/local/android-sdk/tools/keymaps
+RUN touch /usr/local/android-sdk/tools/keymaps/en-us
+
+# Run sshd
+RUN apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+RUN echo "root:$ROOTPASSWORD" | chpasswd
+RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -i 's/PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+ENV NOTVISIBLE "in users profile"
+RUN echo "export VISIBLE=now" >> /etc/profile
+
+# Install socat
+RUN apt-get install -y socat
+
+# symlink android bins
+RUN ln -sv /usr/local/android-sdk/tools/android /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/emulator /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/ddms /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/scheenshot2 /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/monkeyrunner /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/monitor /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/mksdcard /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/uiautomatorviewer /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/tools/traceview /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/platform-tools/adb /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/platform-tools/fastboot /usr/local/bin/
+RUN ln -sv /usr/local/android-sdk/platform-tools/sqlite3 /usr/local/bin/
+
+# Setup DROZER...
+# https://labs.mwrinfosecurity.com/tools/drozer/
+
+# Run as drozer user
+WORKDIR /home/drozer
+
+# Site lists the shasums, however, I'm not sure the best way to integrate the
+# checks here. No real idiomatic way for Dockerfile to do that and most of
+# the examples online use chained commands but we want things to *BREAK* when
+# the sha doesn't match. So far, I can't seem to reliably make Docker not
+# finish the image build process.
+
+# Download the console
+RUN wget -c $DROZER_URL
+
+# Install the console
+RUN dpkg -i $DROZER_DEB
+
+# Download agent
+RUN wget -c $AGENT_URL
+# Keep it version agnostic for other scripts such as install_drozer.py
+RUN mv -v $AGENT_APK drozer-agent.apk
+
+# Port forwarding required by drozer
+RUN echo 'adb forward tcp:31415 tcp:31415' >> /home/drozer/.bashrc
+
+# Alias for Drozer
+RUN echo "alias drozer='drozer console connect'" >> /home/drozer/.bashrc
+
+# add extra scripting
+COPY install_agent.py /home/drozer/install_agent.py
+RUN chmod 755 /home/drozer/install_agent.py
+COPY enable_service.py /home/drozer/enable_service.py
+RUN chmod 755 /home/drozer/enable_service.py
+COPY drozer.py /home/drozer/drozer.py
+RUN chmod 755 /home/drozer/drozer.py
+
+# fix ownerships
+RUN chown -R drozer.drozer /home/drozer
+
+RUN apt-get -y --force-yes install python-pkg-resources=3.3-1ubuntu1
+RUN apt-get -y install python-pip python-setuptools git
+RUN pip install "git+https://github.com/dtmilano/AndroidViewClient.git#egg=androidviewclient"
+RUN apt-get -y install python-pexpect
+
+# Add entrypoint
+COPY entrypoint.sh /home/drozer/entrypoint.sh
+RUN chmod +x /home/drozer/entrypoint.sh
+ENTRYPOINT ["/home/drozer/entrypoint.sh"]

docker/Makefile

@@ -0,0 +1,48 @@
+SHELL := /bin/bash
+ALIAS = "dscanner"
+EXISTS := $(shell docker ps -a -q -f name=$(ALIAS))
+RUNNED := $(shell docker ps -q -f name=$(ALIAS))
+ifneq "$(RUNNED)" ""
+IP := $(shell docker inspect $(ALIAS) | grep "IPAddress\"" | head -n1 | cut -d '"' -f 4)
+endif
+STALE_IMAGES := $(shell docker images | grep "<none>" | awk '{print($$3)}')
+EMULATOR ?= "android-19"
+ARCH ?= "armeabi-v7a"
+
+COLON := :
+
+.PHONY = build clean kill info
+
+all: help
+
+help:
+	@echo "usage: make {help|build|clean|kill|info}"
+	@echo ""
+	@echo "  help    this help screen"
+	@echo "  build   create docker image"
+	@echo "  clean   remove images and containers"
+	@echo "  kill    stop running containers"
+	@echo "  info    details of running container"
+
+build:
+	@docker build -t "dscanner/fdroidserver:latest" .
+
+clean: kill
+	@docker ps -a -q | xargs -n 1 -I {} docker rm -f {}
+ifneq "$(STALE_IMAGES)" ""
+	@docker rmi -f $(STALE_IMAGES)
+endif
+
+kill:
+ifneq "$(RUNNED)" ""
+	@docker kill $(ALIAS)
+endif
+
+info:
+	@docker ps -a -f name=$(ALIAS)
+ifneq "$(RUNNED)" ""
+	$(eval ADBPORT := $(shell docker port $(ALIAS) | grep '5555/tcp' | awk '{split($$3,a,"$(COLON)");print a[2]}'))
+	@echo -e "Use:\n adb kill-server\n adb connect $(IP):$(ADBPORT)"
+else
+	@echo "Run container"
+endif

docker/README.md

@@ -0,0 +1,13 @@
+# dscanner docker image #
+
+Use `make help` for up-to-date instructions.
+
+```
+usage: make {help|build|clean|kill|info}
+
+  help    this help screen
+  build   create docker image
+  clean   remove images and containers
+  kill    stop running containers
+  info    details of running container
+```

docker/drozer.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python2
+
+import pexpect
+import sys
+
+prompt = "dz>"
+target = sys.argv[1]
+
+drozer = pexpect.spawn("drozer console connect")
+drozer.logfile = open("/tmp/drozer_report.log", "w")
+
+
+# start
+drozer.expect(prompt)
+
+
+def send_command(command, target):
+    cmd = "run {0} -a {1}".format(command, target)
+    drozer.sendline(cmd)
+    drozer.expect(prompt)
+
+scanners = [
+    "scanner.misc.native",          # Find native components included in packages
+    #"scanner.misc.readablefiles",   # Find world-readable files in the given folder
+    #"scanner.misc.secretcodes",     # Search for secret codes that can be used from the dialer
+    #"scanner.misc.sflagbinaries",   # Find suid/sgid binaries in the given folder (default is /system).
+    #"scanner.misc.writablefiles",   # Find world-writable files in the given folder
+    "scanner.provider.finduris",    # Search for content providers that can be queried.
+    "scanner.provider.injection",   # Test content providers for SQL injection vulnerabilities.
+    "scanner.provider.sqltables",   # Find tables accessible through SQL injection vulnerabilities.
+    "scanner.provider.traversal"    # Test content providers for basic directory traversal
+]
+
+for scanner in scanners:
+    send_command(scanner, target)

docker/enable_service.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python2
+
+from com.dtmilano.android.viewclient import ViewClient
+
+vc = ViewClient(*ViewClient.connectToDeviceOrExit())
+
+button = vc.findViewWithText("OFF")
+
+if button:
+    (x, y) = button.getXY()
+    button.touch()
+else:
+    print("Button not found. Is the app currently running?")
+    exit()
+
+print("Done!")

docker/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [[ $EMULATOR == "" ]]; then
+    EMULATOR="android-19"
+    echo "Using default emulator $EMULATOR"
+fi
+
+if [[ $ARCH == "" ]]; then
+    ARCH="x86"
+    echo "Using default arch $ARCH"
+fi
+echo EMULATOR  = "Requested API: ${EMULATOR} (${ARCH}) emulator."
+if [[ -n $1 ]]; then
+    echo "Last line of file specified as non-opt/last argument:"
+    tail -1 $1
+fi
+
+# Run sshd
+/usr/sbin/sshd
+adb start-server
+
+# Detect ip and forward ADB ports outside to outside interface
+ip=$(ifconfig  | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}')
+socat tcp-listen:5037,bind=$ip,fork tcp:127.0.0.1:5037 &
+socat tcp-listen:5554,bind=$ip,fork tcp:127.0.0.1:5554 &
+socat tcp-listen:5555,bind=$ip,fork tcp:127.0.0.1:5555 &
+
+# Set up and run emulator
+if [[ $ARCH == *"x86"* ]]
+then
+    EMU="x86"
+else
+    EMU="arm"
+fi
+
+#FASTDROID_VNC_URL="https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/fastdroid-vnc/fastdroid-vnc"
+#wget -c "${FASTDROID_VNC_URL}"
+
+export PATH="${PATH}:/usr/local/android-sdk/tools/:/usr/local/android-sdk/platform-tools/"
+
+echo "no" | android create avd -f -n test -t ${EMULATOR} --abi default/${ARCH}
+echo "no" | emulator64-${EMU} -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0

docker/install_agent.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python2
+
+import os
+from subprocess import call, check_output
+from time import sleep
+
+FNULL = open(os.devnull, 'w')
+
+print("Ensuring device is online")
+call("adb wait-for-device", shell=True)
+
+print("Installing the drozer agent")
+print("If the device just came online it is likely the package manager hasn't booted.")
+print("Will try multiple attempts to install.")
+print("This may need tweaking depending on hardware.")
+
+
+attempts = 0
+time_to_sleep = 30
+
+while attempts < 8:
+    output = check_output('adb shell "pm list packages"', shell=True)
+    print("Checking whether the package manager is up...")
+    if "Could not access the Package Manager" in output:
+        print("Nope. Sleeping for 30 seconds and then trying again.")
+        sleep(time_to_sleep)
+    else:
+        break
+
+time_to_sleep = 5
+attempts = 0
+
+while attempts < 5:
+    sleep(time_to_sleep)
+    try:
+        install_output = check_output("adb install /home/drozer/drozer-agent.apk", shell=True)
+    except Exception:
+        print("Failed. Trying again.")
+        attempts += 1
+    else:
+        attempts += 1
+        if "Error: Could not access the Package Manager" not in install_output:
+            break
+
+print("Install attempted. Checking everything worked")
+
+pm_list_output = check_output('adb shell "pm list packages"', shell=True)
+
+if "com.mwr.dz" not in pm_list_output:
+    print(install_output)
+    exit("APK didn't install properly. Exiting.")
+
+print("Installed ok.")
+
+print("Starting the drozer agent main activity: com.mwr.dz/.activities.MainActivity")
+call('adb shell "am start com.mwr.dz/.activities.MainActivity"', shell=True, stdout=FNULL)
+
+print("Starting the service")
+# start the service
+call("python /home/drozer/enable_service.py", shell=True, stdout=FNULL)
+
+print("Forward dem ports mon.")
+call("adb forward tcp:31415 tcp:31415", shell=True, stdout=FNULL)

docs/fdroid.texi

@@ -57,6 +57,7 @@ Free Documentation License".
 * Update Processing::
 * Build Server::
 * Signing::
+* Vulnerability Scanning::
 * GNU Free Documentation License::
 * Index::
 @end menu
@@ -1697,6 +1698,132 @@ A new key will be generated using these details, for each application that is
 built. (If a specific key is required for a particular application, this system
 can be overridden using the @code{keyaliases} config settings.
 
+@node Vulnerability Scanning
+@chapter Vulnerability Scanning (dscanner)
+
+F-Droid now includes a means of running automated vulnerability scanning
+using @uref{https://github.com/mwrlabs/drozer, Drozer}. This is achieved
+by starting a docker container, with the Android SDK and Emulator
+prepared already, installing drozer into the emulator and scripting the
+knobs to scan any fully built and signed APKs.
+
+Note: if your application is not intended to run within an Android
+emulator, please do not continue with these instructions. At this time,
+the @code{dscanner} feature is fully dependent upon your application
+running properly in an emulated environment.
+
+@section Quick Start
+
+@enumerate
+@item Ensure that your application is a signed release build
+@item @code{fdroid dscanner --init-only} from within the repo
+@item Go for a coffee, this takes a long time and requires approximately
+6 GB of disk space. Once this is complete, you'll be left with a docker
+container running and ready to go.
+@item @code{fdroid dscanner --latest app.pkg.name} from within the repo
+to run drozer on the latest build of @code{app.pkg.name}
+@item If all went well, there should be an ``app.pkg.name_CODE.apk.dscanner''
+file in the repo (next to the original APK file)
+@item When you're all done scanning packages, you can cleanup the docker
+container with: @code{fdroid dscanner --clean-only}
+@end enumerate
+
+You can also run the drozer scan as an optional part of the overall
+@code{fdroid build} operation. This option will trigger a drozer scan of
+all signed APKs found in the repo. See @code{fdroid build --help} for
+more information.
+
+@section Command Line Help
+
+@example
+usage: fdroid dscanner [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]
+
+positional arguments:
+  app_id                app-id with optional versioncode in the form
+                        APPID[:VERCODE]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -v, --verbose         Spew out even more information than normal
+  -q, --quiet           Restrict output to warnings and errors
+  -l, --latest          Scan only the latest version of each package
+  --clean-after         Clean after all scans have finished.
+  --clean-before        Clean before the scans start and rebuild the
+                        container.
+  --clean-only          Clean up all containers and then exit.
+  --init-only           Prepare drozer to run a scan
+  --repo-path REPO_PATH
+                        Override repo path for built APK files.
+@end example
+
+@section From Scratch
+
+Because the docker image used to do the Android Emulator and all of that
+takes a considerable amount of time to prepare, one has been uploaded to
+dockerhub.com for general use. However, the astute researcher will be
+weary of any black boxes and want to build their own black box. This
+section elaborates how to build the docker image yourself.
+
+From within the F-Droid Server source code directory, @code{cd
+./docker/} in order to begin.
+
+Within this directory are the custom scripting used within the docker
+image creation. For conveience, there is a simple Makefile that
+wraps the process of creating images into convenient pieces.
+
+@subsection @code{make help}
+
+@example
+usage: make help|build|clean|kill|info
+
+  help    this help screen
+  build   create docker image
+  clean   remove images and containers
+  kill    stop running containers
+  info    details of running container
+@end example
+
+@subsection @code{make clean}
+
+Stops any running containers (@code{make kill}) and then forcully
+removes them from docker. After that, all images associated are also
+explicitly removed.
+
+Note: this will destroy docker images!
+
+@subsection @code{make build}
+
+Builds the actual docker container, tagged
+``dscanner/fdroidserver:latest'' from the local directory. Obviously
+this is operating with the @code{Dockerfile} to build and tie everything
+together nicely.
+
+@subsection @code{make kill}
+
+@code{docker kill} the container tagged ``dscanner''.
+
+@subsection @code{make info}
+
+Prints some useful information about the currently running dscanner
+container (if it is even running). The output of this command is
+confusing and raw but useful none-the-less. See example output below:
+
+@example
+CONTAINER ID        IMAGE                   COMMAND
+CREATED             STATUS              PORTS
+NAMES
+b90a60afe477        dscanner/fdroidserver   "/home/drozer/entrypo"   20
+minutes ago      Up 20 minutes       0.0.0.0:32779->22/tcp,
+0.0.0.0:32778->5037/tcp, 0.0.0.0:32777->5554/tcp,
+0.0.0.0:32776->5555/tcp, 0.0.0.0:32775->5900/tcp,
+0.0.0.0:32774->5901/tcp   dscanner
+Use:
+ adb kill-server
+ adb connect 172.17.0.2:32776
+@end example
+
+Typical usage is for finding the ``adb connect'' line or the ``ssh''
+port (32779 from the @code{0.0.0.0:32779->22/tcp} note).
 
 @node GNU Free Documentation License
 @appendix GNU Free Documentation License

fdroid

@@ -38,6 +38,7 @@ commands = {
     "rewritemeta": "Rewrite all the metadata files",
     "lint": "Warn about possible metadata errors",
     "scanner": "Scan the source code of a package",
+    "dscanner": "Dynamically scan APKs post build",
     "stats": "Update the stats of the repo",
     "server": "Interact with the repo HTTP server",
     "signindex": "Sign indexes created using update --nosign",

fdroidserver/build.py

@@ -1006,6 +1006,8 @@ def parse_commandline():
                         help="Specify that we're running on the build server")
     parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
                         help="Skip scanning the source code for binaries and other problems")
+    parser.add_argument("--dscanner", action="store_true", default=False,
+                        help="Setup an emulator, install the apk on it and perform a drozer scan")
     parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
                         help="Don't create a source tarball, useful when testing a build")
     parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
@@ -1221,6 +1223,42 @@ def main():
         for fa in failed_apps:
             logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
 
+    # perform a drozer scan of all successful builds
+    if options.dscanner and build_succeeded:
+        from .dscanner import DockerDriver
+
+        docker = DockerDriver()
+
+        try:
+            for app in build_succeeded:
+
+                logging.info("Need to sign the app before we can install it.")
+                subprocess.call("fdroid publish {0}".format(app.id), shell=True)
+
+                apk_path = None
+
+                for f in os.listdir(repo_dir):
+                    if f.endswith('.apk') and f.startswith(app.id):
+                        apk_path = os.path.join(repo_dir, f)
+                        break
+
+                if not apk_path:
+                    raise Exception("No signed APK found at path: {0}".format(apk_path))
+
+                if not os.path.isdir(repo_dir):
+                    exit(1)
+
+                logging.info("Performing Drozer scan on {0}.".format(app))
+                docker.perform_drozer_scan(apk_path, app.id, repo_dir)
+        except Exception as e:
+            logging.error(str(e))
+            logging.error("An exception happened. Making sure to clean up")
+        else:
+            logging.info("Scan succeeded.")
+
+        logging.info("Cleaning up after ourselves.")
+        docker.clean()
+
     logging.info("Finished.")
     if len(build_succeeded) > 0:
         logging.info(str(len(build_succeeded)) + ' builds succeeded')

fdroidserver/dscanner.py

@@ -0,0 +1,482 @@
+#!/usr/bin/env python3
+#
+# dscanner.py - part of the FDroid server tools
+# Copyright (C) 2016-2017 Shawn Gustaw <self@shawngustaw.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import json
+import sys
+from time import sleep
+from argparse import ArgumentParser
+from subprocess import CalledProcessError, check_output
+
+from fdroidserver import common, metadata
+
+try:
+    from docker import Client
+except ImportError:
+    logging.error(("Docker client not installed."
+                   "Install it using pip install docker-py"))
+
+config = None
+options = None
+
+
+class DockerConfig:
+    ALIAS = "dscanner"
+    CONTAINER = "dscanner/fdroidserver"
+    EMULATOR = "android-19"
+    ARCH = "armeabi-v7a"
+
+
+class DockerDriver(object):
+    """
+    Handles all the interactions with the docker container the
+    Android emulator runs in.
+    """
+    class Commands:
+        build = ['docker', 'build', '--no-cache=false', '--pull=true',
+                 '--quiet=false', '--rm=true', '-t',
+                 '{0}:latest'.format(DockerConfig.CONTAINER), '.']
+        run = [
+            'docker', 'run',
+            '-e', '"EMULATOR={0}"'.format(DockerConfig.EMULATOR),
+            '-e', '"ARCH={0}"'.format(DockerConfig.ARCH),
+            '-d', '-P', '--name',
+            '{0}'.format(DockerConfig.ALIAS), '--log-driver=json-file',
+            DockerConfig.CONTAINER]
+        start = ['docker', 'start', '{0}'.format(DockerConfig.ALIAS)]
+        inspect = ['docker', 'inspect', '{0}'.format(DockerConfig.ALIAS)]
+        pm_list = 'adb shell "pm list packages"'
+        install_drozer = "docker exec {0} python /home/drozer/install_agent.py"
+        run_drozer = 'python /home/drozer/drozer.py {0}'
+        copy_to_container = 'docker cp "{0}" {1}:{2}'
+        copy_from_container = 'docker cp {0}:{1} "{2}"'
+
+    def __init__(self, init_only=False, fresh_start=False, clean_only=False):
+        self.container_id = None
+        self.ip_address = None
+
+        self.cli = Client(base_url='unix://var/run/docker.sock')
+
+        if fresh_start or clean_only:
+            self.clean()
+
+        if clean_only:
+            logging.info("Cleaned containers and quitting.")
+            exit(0)
+
+        self.init_docker()
+
+        if init_only:
+            logging.info("Initialized and quitting.")
+            exit(0)
+
+    def _copy_to_container(self, src_path, dest_path):
+        """
+        Copies a file (presumed to be an apk) from src_path
+        to home directory on container.
+        """
+        path = '/home/drozer/{path}.apk'.format(path=dest_path)
+        command = self.Commands.copy_to_container.format(src_path,
+                                                         self.container_id,
+                                                         path)
+
+        try:
+            check_output(command, shell=True)
+        except CalledProcessError as e:
+            logging.error(('Command "{command}" failed with '
+                           'error code {code}'.format(command=command,
+                                                      code=e.returncode)))
+            raise
+
+    def _copy_from_container(self, src_path, dest_path):
+        """
+        Copies a file from src_path on the container to
+        dest_path on the host machine.
+        """
+        command = self.Commands.copy_from_container.format(self.container_id,
+                                                           src_path,
+                                                           dest_path)
+        try:
+            check_output(command, shell=True)
+        except CalledProcessError as e:
+            logging.error(('Command "{command}" failed with '
+                           'error code {code}'.format(command=command,
+                                                      code=e.returncode)))
+            raise
+
+        logging.info("Log stored at {path}".format(path=dest_path))
+
+    def _adb_install_apk(self, apk_path):
+        """
+        Installs an apk on the device running in the container
+        using adb.
+        """
+        logging.info("Attempting to install an apk.")
+        exec_id = self.cli.exec_create(
+            self.container_id, 'adb install {0}'
+            .format(apk_path)
+            )['Id']
+        output = self.cli.exec_start(exec_id).decode('utf-8')
+
+        if "INSTALL_PARSE_FAILED_NO_CERTIFICATES" in output:
+            raise Exception('Install parse failed, no certificates')
+        elif "INSTALL_FAILED_ALREADY_EXISTS" in output:
+            logging.info("APK already installed. Skipping.")
+        elif "Success" not in output:
+            logging.error("APK didn't install properly")
+            return False
+        return True
+
+    def _adb_uninstall_apk(self, app_id):
+        """
+        Uninstalls an application from the device running in the container
+        via its app_id.
+        """
+        logging.info(
+            "Uninstalling {app_id} from the emulator."
+            .format(app_id=app_id)
+            )
+        exec_id = self.cli.exec_create(
+            self.container_id,
+            'adb uninstall {0}'.format(app_id)
+            )['Id']
+        output = self.cli.exec_start(exec_id).decode('utf-8')
+
+        if 'Success' in output:
+            logging.info("Successfully uninstalled.")
+
+        return True
+
+    def _verify_apk_install(self, app_id):
+        """
+        Checks that the app_id is installed on the device running in the
+        container.
+        """
+        logging.info(
+            "Verifying {app} is installed on the device."
+            .format(app=app_id)
+            )
+        exec_id = self.cli.exec_create(
+            self.container_id, self.Commands.pm_list
+            )['Id']
+        output = self.cli.exec_start(exec_id).decode('utf-8')
+
+        if ("Could not access the Package Manager" in output or
+                "device offline" in output):
+            logging.info("Device or package manager isn't up")
+
+        if app_id.split('_')[0] in output:   # TODO: this is a temporary fix
+            logging.info("{app} is installed.".format(app=app_id))
+            return True
+
+        logging.error("APK not found in packages list on emulator.")
+
+    def _delete_file(self, path):
+        """
+        Deletes file off the container to preserve space if scanning many apps
+        """
+        command = "rm {path}".format(path=path)
+        exec_id = self.cli.exec_create(self.container_id, command)['Id']
+        logging.info("Deleting {path} on the container.".format(path=path))
+        self.cli.exec_start(exec_id)
+
+    def _install_apk(self, apk_path, app_id):
+        """
+        Installs apk found at apk_path on the emulator. Will then
+        verify it installed properly by looking up its app_id in
+        the package manager.
+        """
+        if not all([self.container_id, self.ip_address]):
+            # TODO: maybe have this fail nicely
+            raise Exception("Went to install apk and couldn't find container")
+
+        path = "/home/drozer/{app_id}.apk".format(app_id=app_id)
+        self._copy_to_container(apk_path, app_id)
+        self._adb_install_apk(path)
+        self._verify_apk_install(app_id)
+        self._delete_file(path)
+
+    def _install_drozer(self):
+        """
+        Performs all the initialization of drozer within the emulator.
+        """
+        logging.info("Attempting to install com.mwr.dz on the emulator")
+        logging.info("This could take a while so be patient...")
+        logging.info(("We need to wait for the device to boot AND"
+                      " the package manager to come online."))
+        command = self.Commands.install_drozer.format(self.container_id)
+        try:
+            output = check_output(command,
+                                  shell=True).decode('utf-8')
+        except CalledProcessError as e:
+            logging.error(('Command "{command}" failed with '
+                           'error code {code}'.format(command=command,
+                                                      code=e.returncode)))
+            raise
+
+        if 'Installed ok' in output:
+            return True
+
+    def _run_drozer_scan(self, app):
+        """
+        Runs the drozer agent which connects to the app running
+        on the emulator.
+        """
+        logging.info("Running the drozer agent")
+        exec_id = self.cli.exec_create(
+            self.container_id,
+            self.Commands.run_drozer.format(app)
+            )['Id']
+        self.cli.exec_start(exec_id)
+
+    def _container_is_running(self):
+        """
+        Checks whether the emulator container is running.
+        """
+        for container in self.cli.containers():
+            if DockerConfig.ALIAS in container['Image']:
+                return True
+
+    def _docker_image_exists(self):
+        """
+        Check whether the docker image exists already.
+        If this returns false we'll need to build the image
+        from the DockerFile.
+        """
+        for image in self.cli.images():
+            for tag in image['RepoTags']:
+                if DockerConfig.ALIAS in tag:
+                    return True
+
+    _image_queue = {}
+
+    def _build_docker_image(self):
+        """
+        Builds the docker container so we can run the android emulator
+        inside it.
+        """
+        logging.info("Pulling the container from docker hub")
+        logging.info("Image is roughly 5 GB so be patient")
+
+        logging.info("(Progress output is slow and requires a tty.)")
+        # we pause briefly to narrow race condition windows of opportunity
+        sleep(1)
+
+        is_a_tty = os.isatty(sys.stdout.fileno())
+
+        for output in self.cli.pull(
+                DockerConfig.CONTAINER,
+                stream=True,
+                tag="latest"):
+            if not is_a_tty:
+                # run silent, run quick
+                continue
+            try:
+                p = json.loads(output.decode('utf-8'))
+                p_id = p['id']
+                self._image_queue[p_id] = p
+                t, c, j = 1, 1, 0
+                for k in sorted(self._image_queue):
+                    j += 1
+                    v = self._image_queue[k]
+                    vd = v['progressDetail']
+                    t += vd['total']
+                    c += vd['current']
+                msg = "\rDownloading: {0}/{1} {2}% [{3} jobs]"
+                msg = msg.format(c, t, int(c / t * 100), j)
+                sys.stdout.write(msg)
+                sys.stdout.flush()
+            except:
+                pass
+        print("\nDONE!\n")
+
+    def _verify_apk_exists(self, full_apk_path):
+        """
+        Verifies that the apk path we have is actually a file.
+        """
+        return os.path.isfile(full_apk_path)
+
+    def init_docker(self):
+        """
+        Perform all the initialization required before a drozer scan.
+        1. build the image
+        2. run the container
+        3. install drozer and enable the service within the app
+        """
+        built = self._docker_image_exists()
+
+        if not built:
+            self._build_docker_image()
+
+        running = self._container_is_running()
+
+        if not running:
+            logging.info('Trying to run container...')
+            try:
+                check_output(self.Commands.run)
+            except CalledProcessError as e:
+                logging.error((
+                    'Command "{command}" failed with error code {code}'
+                    .format(command=self.Commands.run, code=e.returncode)
+                    ))
+            running = self._container_is_running()
+
+        if not running:
+            logging.info('Trying to start container...')
+            try:
+                check_output(self.Commands.start)
+            except CalledProcessError as e:
+                logging.error((
+                    'Command "{command}" failed with error code {code}'
+                    .format(command=self.Commands.run, code=e.returncode)
+                    ))
+            running = self._container_is_running()
+
+        if not running:
+            raise Exception("Running container not found, critical error.")
+
+        containers = self.cli.containers()
+
+        for container in containers:
+            if DockerConfig.ALIAS in container['Image']:
+                self.container_id = container['Id']
+                n = container['NetworkSettings']['Networks']
+                self.ip_address = n['bridge']['IPAddress']
+                break
+
+        if not self.container_id or not self.ip_address:
+            logging.error("No ip address or container id found.")
+            exit(1)
+
+        if self._verify_apk_install('com.mwr.dz'):
+            return
+
+        self._install_drozer()
+
+    def clean(self):
+        """
+        Clean up all the containers made by this script.
+        Should be run after the drozer scan completes.
+        """
+        for container in self.cli.containers():
+            if DockerConfig.ALIAS in container['Image']:
+                logging.info("Removing container {0}".format(container['Id']))
+                self.cli.remove_container(container['Id'], force=True)
+
+    def perform_drozer_scan(self, apk_path, app_id):
+        """
+        Entrypoint for scanning an android app. Performs the following steps:
+        1. installs an apk on the device
+        2. runs a drozer scan
+        3. copies the report off the container
+        4. uninstalls the apk to save space on the device
+        """
+        self._install_apk(apk_path, app_id)
+        logging.info("Running the drozer scan.")
+        self._run_drozer_scan(app_id)
+        logging.info("Scan finished. Moving the report off the container")
+        dest = apk_path + '.drozer'
+        self._copy_from_container('/tmp/drozer_report.log', dest)
+        self._adb_uninstall_apk(app_id)
+
+
+def main():
+    global config, options
+
+    # Parse command line...
+    parser = ArgumentParser(
+        usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
+        )
+    common.setup_global_opts(parser)
+
+    parser.add_argument(
+        "app_id", nargs='*',
+        help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument(
+        "-l", "--latest", action="store_true", default=False,
+        help="Scan only the latest version of each package")
+    parser.add_argument(
+        "--clean-after", default=False, action='store_true',
+        help="Clean after all scans have finished")
+    parser.add_argument(
+        "--clean-before", default=False, action='store_true',
+        help="Clean before the scans start and rebuild the container")
+    parser.add_argument(
+        "--clean-only", default=False, action='store_true',
+        help="Clean up all containers and then exit")
+    parser.add_argument(
+        "--init-only", default=False, action='store_true',
+        help="Prepare drozer to run a scan")
+    parser.add_argument(
+        "--repo-path", default="repo", action="store",
+        help="Override path for repo APKs (default: ./repo)")
+
+    options = parser.parse_args()
+    config = common.read_config(options)
+
+    if not os.path.isdir(options.repo_path):
+        sys.stderr.write("repo-path not found: \"" + options.repo_path + "\"")
+        exit(1)
+
+    # Read all app and srclib metadata
+    allapps = metadata.read_metadata()
+    apps = common.read_app_args(options.app_id, allapps, True)
+
+    docker = DockerDriver(
+        init_only=options.init_only,
+        fresh_start=options.clean_before,
+        clean_only=options.clean_only
+    )
+
+    if options.clean_before:
+        docker.clean()
+
+    if options.clean_only:
+        exit(0)
+
+    for app_id, app in apps.items():
+        vercode = 0
+        if ':' in app_id:
+            vercode = app_id.split(':')[1]
+        for build in reversed(app.builds):
+            if build.disable:
+                continue
+            if options.latest or vercode == 0 or build.vercode == vercode:
+                app.builds = [build]
+                break
+            continue
+        continue
+
+    for app_id, app in apps.items():
+        for build in app.builds:
+            apks = []
+            for f in os.listdir(options.repo_path):
+                n = "%v_%v.apk".format(app_id, build.vercode)
+                if f == n:
+                    apks.append(f)
+            for apk in sorted(apks):
+                apk_path = os.path.join(options.repo_path, apk)
+                docker.perform_drozer_scan(apk_path, app.id)
+
+    if options.clean_after:
+        docker.clean()
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -36,7 +36,8 @@ setup(name='fdroidserver',
           'pyasn1',
           'pyasn1-modules',
           'PyYAML',
-          'requests',
+          'requests < 2.11',
+          'docker-py == 1.9.0',
       ],
       classifiers=[
           'Development Status :: 3 - Alpha',