Merge branch 'binary-transparency' into 'master'

new repo-wide config option for a "Binary Transparency" log based on git

See merge request !226

examples/config.py

@@ -238,6 +238,13 @@ The repository of older versions of applications from the main demo repository.
 # wiki_user = "login"
 # wiki_password = "1234"
 
+# Keep a log of all generated index files in a git repo to provide a
+# "binary transparency" log for anyone to check the history of the
+# binaries that are published.  This is in the form of a "git remote",
+# which this machine where `fdroid update` is run has already been
+# configured to allow push access (e.g. ssh key, username/password, etc)
+# binary_transparency_remote = "git@gitlab.com:fdroid/binary-transparency-log.git"
+
 # Only set this to true when running a repository where you want to generate
 # stats, and only then on the master build servers, not a development
 # machine.

fdroidserver/common.py

@@ -1998,6 +1998,23 @@ def compare_apks(apk1, apk2, tmp_dir):
     trying to do the comparison.
     """
 
+    absapk1 = os.path.abspath(apk1)
+    absapk2 = os.path.abspath(apk2)
+
+    # try to find diffoscope in the path, if it hasn't been manually configed
+    if 'diffoscope' not in config:
+        tmp = find_command('diffoscope')
+        if tmp is not None:
+            config['diffoscope'] = tmp
+    if 'diffoscope' in config:
+        htmlfile = absapk1 + '.diffoscope.html'
+        textfile = absapk1 + '.diffoscope.txt'
+        if subprocess.call([config['diffoscope'],
+                            '--max-report-size', '12345678', '--max-diff-block-lines', '100',
+                            '--html', htmlfile, '--text', textfile,
+                            absapk1, absapk2]) != 0:
+            return("Failed to unpack " + apk1)
+
     apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4]))  # trim .apk
     apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4]))  # trim .apk
     for d in [apk1dir, apk2dir]:

fdroidserver/server.py

@@ -279,6 +279,22 @@ def upload_to_virustotal(repo_section, vt_apikey):
             logging.info(response['verbose_msg'] + " " + response['permalink'])
 
 
+def push_binary_transparency(binary_transparency_remote):
+    '''push the binary transparency git repo to the specifed remote'''
+    import git
+
+    repo = git.Repo('binary_transparency_log')
+    pushremote = None
+    for remote in repo.remotes:
+        if remote.url == binary_transparency_remote:
+            pushremote = remote
+            break
+
+    if not pushremote:
+        pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote)
+    pushremote.push('master')
+
+
 def main():
     global config, options
 
@@ -414,6 +430,11 @@ def main():
                 upload_to_android_observatory(repo_section)
             if config.get('virustotal_apikey'):
                 upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
+
+            binary_transparency_remote = config.get('binary_transparency_remote')
+            if binary_transparency_remote:
+                push_binary_transparency(binary_transparency_remote)
+
     sys.exit(0)
 
 

fdroidserver/update.py

@@ -23,11 +23,13 @@ import sys
 import os
 import shutil
 import glob
+import json
 import re
 import socket
 import zipfile
 import hashlib
 import pickle
+import platform
 import urllib.parse
 from datetime import datetime, timedelta
 from xml.dom.minidom import Document
@@ -1439,6 +1441,80 @@ def add_apks_to_per_app_repos(repodir, apks):
             shutil.copy(apkascpath, apk['per_app_repo'])
 
 
+def make_binary_transparency_log(repodirs):
+    '''Log the indexes in a standalone git repo to serve as a "binary
+    transparency" log.
+
+    see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
+
+    '''
+
+    import git
+    btrepo = 'binary_transparency'
+    if os.path.exists(os.path.join(btrepo, '.git')):
+        gitrepo = git.Repo(btrepo)
+    else:
+        if not os.path.exists(btrepo):
+            os.mkdir(btrepo)
+        gitrepo = git.Repo.init(btrepo)
+
+        gitconfig = gitrepo.config_writer()
+        gitconfig.set_value('user', 'name', 'fdroid update')
+        gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
+
+        url = config['repo_url'].rstrip('/')
+        with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
+            fp.write("""
+# Binary Transparency Log for %s
+
+""" % url[:url.rindex('/')])  # strip '/repo'
+        gitrepo.index.add(['README.md', ])
+        gitrepo.index.commit('add README')
+
+    for repodir in repodirs:
+        cpdir = os.path.join(btrepo, repodir)
+        if not os.path.exists(cpdir):
+            os.mkdir(cpdir)
+        for f in ('index.xml', ):
+            dest = os.path.join(cpdir, f)
+            shutil.copyfile(os.path.join(repodir, f), dest)
+            gitrepo.index.add([os.path.join(repodir, f), ])
+        for f in ('index.jar', ):
+            repof = os.path.join(repodir, f)
+            dest = os.path.join(cpdir, f)
+            jarin = zipfile.ZipFile(repof, 'r')
+            jarout = zipfile.ZipFile(dest, 'w')
+            for info in jarin.infolist():
+                if info.filename.startswith('META-INF/'):
+                    jarout.writestr(info, jarin.read(info.filename))
+            jarout.close()
+            jarin.close()
+            gitrepo.index.add([repof, ])
+
+        files = []
+        for root, dirs, filenames in os.walk(repodir):
+            for f in filenames:
+                files.append(os.path.relpath(os.path.join(root, f), repodir))
+        output = collections.OrderedDict()
+        for f in sorted(files):
+            repofile = os.path.join(repodir, f)
+            stat = os.stat(repofile)
+            output[f] = (
+                stat.st_size,
+                stat.st_ctime_ns,
+                stat.st_mtime_ns,
+                stat.st_mode,
+                stat.st_uid,
+                stat.st_gid,
+            )
+        fslogfile = os.path.join(cpdir, 'filesystemlog.json')
+        with open(fslogfile, 'w') as fp:
+            json.dump(output, fp, indent=2)
+        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
+
+    gitrepo.index.commit('fdroid update')
+
+
 config = None
 options = None
 
@@ -1678,6 +1754,9 @@ def main():
     if len(repodirs) > 1:
         make_index(apps, sortedids, archapks, repodirs[1], True)
 
+    if config.get('binary_transparency_remote'):
+        make_binary_transparency_log(repodirs)
+
     if config['update_stats']:
 
         # Update known apks info...

tests/run-tests

@@ -553,6 +553,27 @@ fi
 set -e
 
 
+#------------------------------------------------------------------------------#
+echo_header "copy tests/repo, update with binary transparency log"
+
+REPOROOT=`create_test_dir`
+GNUPGHOME=$REPOROOT/gnupghome
+KEYSTORE=$WORKSPACE/tests/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py
+echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
+$fdroid update --verbose --pretty
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+cd binary_transparency
+[ `git rev-list --count HEAD` == "2" ]
+
+
 #------------------------------------------------------------------------------#
 echo_header "setup a new repo with keystore with APK, update, then without key"