Merge branch 'morebintrans' into 'master'

`fdroid btlog` for anyone to setup a binary transparency log for any repo

See merge request !243

completion/bash-completion

@@ -236,6 +236,12 @@ __complete_verify() {
 	esac
 }
 
+__complete_btlog() {
+	opts="-u"
+	lopts="--git-remote --git-repo --url"
+	__complete_options
+}
+
 __complete_stats() {
 	opts="-v -q -d"
 	lopts="--verbose --quiet --download"

fdroid

@@ -42,6 +42,7 @@ commands = {
     "stats": "Update the stats of the repo",
     "server": "Interact with the repo HTTP server",
     "signindex": "Sign indexes created using update --nosign",
+    "btlog": "Update the binary transparency log for a URL",
 }
 
 

fdroidserver/btlog.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+#
+# btlog.py - part of the FDroid server tools
+# Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This is for creating a binary transparency log in a git repo for any
+# F-Droid repo accessible via HTTP.  It is meant to run very often,
+# even once a minute in a cronjob, so it uses HEAD requests and the
+# HTTP ETag to check if the file has changed.  HEAD requests should
+# not count against the download counts.  This pattern of a HEAD then
+# a GET is what fdroidclient uses to avoid ETags being abused as
+# cookies. This also uses the same HTTP User Agent as the F-Droid
+# client app so its not easy for the server to distinguish this from
+# the F-Droid client.
+
+
+import collections
+import git
+import glob
+import os
+import json
+import logging
+import requests
+import shutil
+import sys
+import tempfile
+import xml.dom.minidom
+import zipfile
+from argparse import ArgumentParser
+
+from . import common
+
+
+options = None
+
+
+def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
+                                 url=None,
+                                 commit_title='fdroid update',
+                                 git_remote=None):
+    '''Log the indexes in a standalone git repo to serve as a "binary
+    transparency" log.
+
+    see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
+
+    '''
+
+    logging.info('Committing indexes to ' + btrepo)
+    if os.path.exists(os.path.join(btrepo, '.git')):
+        gitrepo = git.Repo(btrepo)
+    else:
+        if not os.path.exists(btrepo):
+            os.mkdir(btrepo)
+        gitrepo = git.Repo.init(btrepo)
+
+        if not url:
+            url = common.config['repo_url'].rstrip('/')
+        with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
+            fp.write("""
+# Binary Transparency Log for %s
+
+This is a log of the signed app index metadata.  This is stored in a
+git repo, which serves as an imperfect append-only storage mechanism.
+People can then check that any file that they received from that
+F-Droid repository was a publicly released file.
+
+For more info on this idea:
+* https://wiki.mozilla.org/Security/Binary_Transparency
+""" % url[:url.rindex('/')])  # strip '/repo'
+        gitrepo.index.add(['README.md', ])
+        gitrepo.index.commit('add README')
+
+    for repodir in repodirs:
+        cpdir = os.path.join(btrepo, repodir)
+        if not os.path.exists(cpdir):
+            os.mkdir(cpdir)
+        for f in ('index.xml', 'index-v1.json'):
+            repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
+            dest = os.path.join(cpdir, f)
+            if f.endswith('.xml'):
+                doc = xml.dom.minidom.parse(repof)
+                output = doc.toprettyxml(encoding='utf-8')
+                with open(dest, 'wb') as f:
+                    f.write(output)
+            elif f.endswith('.json'):
+                with open(repof) as fp:
+                    output = json.load(fp, object_pairs_hook=collections.OrderedDict)
+                with open(dest, 'w') as fp:
+                    json.dump(output, fp, indent=2)
+            gitrepo.index.add([repof, ])
+        for f in ('index.jar', 'index-v1.jar'):
+            repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
+            dest = os.path.join(cpdir, f)
+            jarin = zipfile.ZipFile(repof, 'r')
+            jarout = zipfile.ZipFile(dest, 'w')
+            for info in jarin.infolist():
+                if info.filename.startswith('META-INF/'):
+                    jarout.writestr(info, jarin.read(info.filename))
+            jarout.close()
+            jarin.close()
+            gitrepo.index.add([repof, ])
+
+        files = []
+        for root, dirs, filenames in os.walk(repodir):
+            for f in filenames:
+                files.append(os.path.relpath(os.path.join(root, f), repodir))
+        output = collections.OrderedDict()
+        for f in sorted(files):
+            repofile = os.path.join(repodir, f)
+            stat = os.stat(repofile)
+            output[f] = (
+                stat.st_size,
+                stat.st_ctime_ns,
+                stat.st_mtime_ns,
+                stat.st_mode,
+                stat.st_uid,
+                stat.st_gid,
+            )
+        fslogfile = os.path.join(cpdir, 'filesystemlog.json')
+        with open(fslogfile, 'w') as fp:
+            json.dump(output, fp, indent=2)
+        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
+
+        for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
+            gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
+
+    gitrepo.index.commit(commit_title)
+    if git_remote:
+        logging.info('Pushing binary transparency log to ' + git_remote)
+        origin = git.remote.Remote(gitrepo, 'origin')
+        if origin in gitrepo.remotes:
+            origin = gitrepo.remote('origin')
+            if 'set_url' in dir(origin):  # added in GitPython 2.x
+                origin.set_url(git_remote)
+        else:
+            origin = gitrepo.create_remote('origin', git_remote)
+        origin.fetch()
+        origin.push('master')
+
+
+def main():
+    global options
+
+    parser = ArgumentParser(usage="%(prog)s [options]")
+    common.setup_global_opts(parser)
+    parser.add_argument("--git-repo",
+                        default=os.path.join(os.getcwd(), 'binary_transparency'),
+                        help="Path to the git repo to use as the log")
+    parser.add_argument("-u", "--url", default='https://f-droid.org',
+                        help="The base URL for the repo to log (default: https://f-droid.org)")
+    parser.add_argument("--git-remote", default=None,
+                        help="Push the log to this git remote repository")
+    options = parser.parse_args()
+
+    if options.verbose:
+        logging.getLogger("requests").setLevel(logging.INFO)
+        logging.getLogger("urllib3").setLevel(logging.INFO)
+    else:
+        logging.getLogger("requests").setLevel(logging.WARNING)
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+    if not os.path.exists(options.git_repo):
+        logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo')
+        sys.exit(1)
+
+    session = requests.Session()
+
+    new_files = False
+    repodirs = ('repo', 'archive')
+    tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-')
+    for repodir in repodirs:
+        # TODO read HTTP headers for etag from git repo
+        tempdir = os.path.join(tempdirbase, repodir)
+        os.makedirs(tempdir, exist_ok=True)
+        gitrepodir = os.path.join(options.git_repo, repodir)
+        os.makedirs(gitrepodir, exist_ok=True)
+        for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'):
+            dlfile = os.path.join(tempdir, f)
+            dlurl = options.url + '/' + repodir + '/' + f
+            http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
+
+            headers = {
+                'User-Agent': 'F-Droid 0.102.3'
+            }
+            if os.path.exists(http_headers_file):
+                with open(http_headers_file) as fp:
+                    etag = json.load(fp)['ETag']
+
+            r = session.head(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code != 200:
+                logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl)
+                continue
+            if etag and etag == r.headers.get('ETag'):
+                logging.debug('ETag matches, did not download ' + dlurl)
+                continue
+
+            r = session.get(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code == 200:
+                with open(dlfile, 'wb') as f:
+                    for chunk in r:
+                        f.write(chunk)
+
+                dump = dict()
+                for k, v in r.headers.items():
+                    dump[k] = v
+                with open(http_headers_file, 'w') as fp:
+                    json.dump(dump, fp, indent=2, sort_keys=True)
+                new_files = True
+
+    if new_files:
+        os.chdir(tempdirbase)
+        make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog',
+                                     git_remote=options.git_remote)
+    shutil.rmtree(tempdirbase, ignore_errors=True)
+
+if __name__ == "__main__":
+    main()

fdroidserver/index.py

@@ -220,7 +220,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
     json_name = 'index-v1.json'
     index_file = os.path.join(repodir, json_name)
     with open(index_file, 'w') as fp:
-        json.dump(output, fp, default=_index_encoder_default)
+        if common.options.pretty:
+            json.dump(output, fp, default=_index_encoder_default, indent=2)
+        else:
+            json.dump(output, fp, default=_index_encoder_default)
 
     if common.options.nosign:
         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')

fdroidserver/update.py

@@ -23,13 +23,11 @@ import sys
 import os
 import shutil
 import glob
-import json
 import re
 import socket
 import zipfile
 import hashlib
 import pickle
-import platform
 from datetime import datetime, timedelta
 from argparse import ArgumentParser
 
@@ -39,6 +37,7 @@ from binascii import hexlify
 from PIL import Image
 import logging
 
+from . import btlog
 from . import common
 from . import index
 from . import metadata
@@ -1212,80 +1211,6 @@ def add_apks_to_per_app_repos(repodir, apks):
             shutil.copy(apkascpath, apk['per_app_repo'])
 
 
-def make_binary_transparency_log(repodirs):
-    '''Log the indexes in a standalone git repo to serve as a "binary
-    transparency" log.
-
-    see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
-
-    '''
-
-    import git
-    btrepo = 'binary_transparency'
-    if os.path.exists(os.path.join(btrepo, '.git')):
-        gitrepo = git.Repo(btrepo)
-    else:
-        if not os.path.exists(btrepo):
-            os.mkdir(btrepo)
-        gitrepo = git.Repo.init(btrepo)
-
-        gitconfig = gitrepo.config_writer()
-        gitconfig.set_value('user', 'name', 'fdroid update')
-        gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
-
-        url = config['repo_url'].rstrip('/')
-        with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
-            fp.write("""
-# Binary Transparency Log for %s
-
-""" % url[:url.rindex('/')])  # strip '/repo'
-        gitrepo.index.add(['README.md', ])
-        gitrepo.index.commit('add README')
-
-    for repodir in repodirs:
-        cpdir = os.path.join(btrepo, repodir)
-        if not os.path.exists(cpdir):
-            os.mkdir(cpdir)
-        for f in ('index.xml', 'index-v1.json'):
-            dest = os.path.join(cpdir, f)
-            shutil.copyfile(os.path.join(repodir, f), dest)
-            gitrepo.index.add([os.path.join(repodir, f), ])
-        for f in ('index.jar', 'index-v1.jar'):
-            repof = os.path.join(repodir, f)
-            dest = os.path.join(cpdir, f)
-            jarin = zipfile.ZipFile(repof, 'r')
-            jarout = zipfile.ZipFile(dest, 'w')
-            for info in jarin.infolist():
-                if info.filename.startswith('META-INF/'):
-                    jarout.writestr(info, jarin.read(info.filename))
-            jarout.close()
-            jarin.close()
-            gitrepo.index.add([repof, ])
-
-        files = []
-        for root, dirs, filenames in os.walk(repodir):
-            for f in filenames:
-                files.append(os.path.relpath(os.path.join(root, f), repodir))
-        output = collections.OrderedDict()
-        for f in sorted(files):
-            repofile = os.path.join(repodir, f)
-            stat = os.stat(repofile)
-            output[f] = (
-                stat.st_size,
-                stat.st_ctime_ns,
-                stat.st_mtime_ns,
-                stat.st_mode,
-                stat.st_uid,
-                stat.st_gid,
-            )
-        fslogfile = os.path.join(cpdir, 'filesystemlog.json')
-        with open(fslogfile, 'w') as fp:
-            json.dump(output, fp, indent=2)
-        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
-
-    gitrepo.index.commit('fdroid update')
-
-
 config = None
 options = None
 
@@ -1483,8 +1408,9 @@ def main():
     if len(repodirs) > 1:
         index.make(apps, sortedids, archapks, repodirs[1], True)
 
-    if config.get('binary_transparency_remote'):
+    git_remote = config.get('binary_transparency_remote')
-        make_binary_transparency_log(repodirs)
+    if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
+        btlog.make_binary_transparency_log(repodirs, git_remote=git_remote)
 
     if config['update_stats']:
         # Update known apks info...

tests/run-tests

@@ -592,12 +592,15 @@ echo_header "copy tests/repo, update with binary transparency log"
 REPOROOT=`create_test_dir`
 GNUPGHOME=$REPOROOT/gnupghome
 KEYSTORE=$WORKSPACE/tests/keystore.jks
+mkdir $REPOROOT/git_remote
+cd $REPOROOT/git_remote
+git init --bare
 cd $REPOROOT
 $fdroid init --keystore $KEYSTORE --repo-keyalias=sova
 cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
 echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
 echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
-echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py
+echo "binary_transparency_remote = '$REPOROOT/git_remote'" >> config.py
 echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
 $fdroid update --verbose --pretty
 test -e repo/index.xml
@@ -606,6 +609,8 @@ test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 cd binary_transparency
 [ `git rev-list --count HEAD` == "2" ]
+cd $REPOROOT/git_remote
+[ `git rev-list --count HEAD` == "2" ]
 
 
 #------------------------------------------------------------------------------#