diff --git a/completion/bash-completion b/completion/bash-completion index bdaa2cbb..48352447 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -236,6 +236,12 @@ __complete_verify() { esac } +__complete_btlog() { + opts="-u" + lopts="--git-remote --git-repo --url" + __complete_options +} + __complete_stats() { opts="-v -q -d" lopts="--verbose --quiet --download" diff --git a/fdroid b/fdroid index feea104a..bc1655b9 100755 --- a/fdroid +++ b/fdroid @@ -42,6 +42,7 @@ commands = { "stats": "Update the stats of the repo", "server": "Interact with the repo HTTP server", "signindex": "Sign indexes created using update --nosign", + "btlog": "Update the binary transparency log for a URL", } diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py new file mode 100755 index 00000000..94d4a2cb --- /dev/null +++ b/fdroidserver/btlog.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# +# btlog.py - part of the FDroid server tools +# Copyright (C) 2017, Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# This is for creating a binary transparency log in a git repo for any +# F-Droid repo accessible via HTTP. It is meant to run very often, +# even once a minute in a cronjob, so it uses HEAD requests and the +# HTTP ETag to check if the file has changed. HEAD requests should +# not count against the download counts. This pattern of a HEAD then +# a GET is what fdroidclient uses to avoid ETags being abused as +# cookies. This also uses the same HTTP User Agent as the F-Droid +# client app so its not easy for the server to distinguish this from +# the F-Droid client. + + +import collections +import git +import glob +import os +import json +import logging +import requests +import shutil +import sys +import tempfile +import xml.dom.minidom +import zipfile +from argparse import ArgumentParser + +from . import common + + +options = None + + +def make_binary_transparency_log(repodirs, btrepo='binary_transparency', + url=None, + commit_title='fdroid update', + git_remote=None): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + + logging.info('Committing indexes to ' + btrepo) + if os.path.exists(os.path.join(btrepo, '.git')): + gitrepo = git.Repo(btrepo) + else: + if not os.path.exists(btrepo): + os.mkdir(btrepo) + gitrepo = git.Repo.init(btrepo) + + if not url: + url = common.config['repo_url'].rstrip('/') + with open(os.path.join(btrepo, 'README.md'), 'w') as fp: + fp.write(""" +# Binary Transparency Log for %s + +This is a log of the signed app index metadata. This is stored in a +git repo, which serves as an imperfect append-only storage mechanism. +People can then check that any file that they received from that +F-Droid repository was a publicly released file. + +For more info on this idea: +* https://wiki.mozilla.org/Security/Binary_Transparency +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) + gitrepo.index.commit('add README') + + for repodir in repodirs: + cpdir = os.path.join(btrepo, repodir) + if not os.path.exists(cpdir): + os.mkdir(cpdir) + for f in ('index.xml', 'index-v1.json'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue + dest = os.path.join(cpdir, f) + if f.endswith('.xml'): + doc = xml.dom.minidom.parse(repof) + output = doc.toprettyxml(encoding='utf-8') + with open(dest, 'wb') as f: + f.write(output) + elif f.endswith('.json'): + with open(repof) as fp: + output = json.load(fp, object_pairs_hook=collections.OrderedDict) + with open(dest, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([repof, ]) + for f in ('index.jar', 'index-v1.jar'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue + dest = os.path.join(cpdir, f) + jarin = zipfile.ZipFile(repof, 'r') + jarout = zipfile.ZipFile(dest, 'w') + for info in jarin.infolist(): + if info.filename.startswith('META-INF/'): + jarout.writestr(info, jarin.read(info.filename)) + jarout.close() + jarin.close() + gitrepo.index.add([repof, ]) + + files = [] + for root, dirs, filenames in os.walk(repodir): + for f in filenames: + files.append(os.path.relpath(os.path.join(root, f), repodir)) + output = collections.OrderedDict() + for f in sorted(files): + repofile = os.path.join(repodir, f) + stat = os.stat(repofile) + output[f] = ( + stat.st_size, + stat.st_ctime_ns, + stat.st_mtime_ns, + stat.st_mode, + stat.st_uid, + stat.st_gid, + ) + fslogfile = os.path.join(cpdir, 'filesystemlog.json') + with open(fslogfile, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + + for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): + gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) + + gitrepo.index.commit(commit_title) + if git_remote: + logging.info('Pushing binary transparency log to ' + git_remote) + origin = git.remote.Remote(gitrepo, 'origin') + if origin in gitrepo.remotes: + origin = gitrepo.remote('origin') + if 'set_url' in dir(origin): # added in GitPython 2.x + origin.set_url(git_remote) + else: + origin = gitrepo.create_remote('origin', git_remote) + origin.fetch() + origin.push('master') + + +def main(): + global options + + parser = ArgumentParser(usage="%(prog)s [options]") + common.setup_global_opts(parser) + parser.add_argument("--git-repo", + default=os.path.join(os.getcwd(), 'binary_transparency'), + help="Path to the git repo to use as the log") + parser.add_argument("-u", "--url", default='https://f-droid.org', + help="The base URL for the repo to log (default: https://f-droid.org)") + parser.add_argument("--git-remote", default=None, + help="Push the log to this git remote repository") + options = parser.parse_args() + + if options.verbose: + logging.getLogger("requests").setLevel(logging.INFO) + logging.getLogger("urllib3").setLevel(logging.INFO) + else: + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + if not os.path.exists(options.git_repo): + logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo') + sys.exit(1) + + session = requests.Session() + + new_files = False + repodirs = ('repo', 'archive') + tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-') + for repodir in repodirs: + # TODO read HTTP headers for etag from git repo + tempdir = os.path.join(tempdirbase, repodir) + os.makedirs(tempdir, exist_ok=True) + gitrepodir = os.path.join(options.git_repo, repodir) + os.makedirs(gitrepodir, exist_ok=True) + for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'): + dlfile = os.path.join(tempdir, f) + dlurl = options.url + '/' + repodir + '/' + f + http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json') + + headers = { + 'User-Agent': 'F-Droid 0.102.3' + } + if os.path.exists(http_headers_file): + with open(http_headers_file) as fp: + etag = json.load(fp)['ETag'] + + r = session.head(dlurl, headers=headers, allow_redirects=False) + if r.status_code != 200: + logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl) + continue + if etag and etag == r.headers.get('ETag'): + logging.debug('ETag matches, did not download ' + dlurl) + continue + + r = session.get(dlurl, headers=headers, allow_redirects=False) + if r.status_code == 200: + with open(dlfile, 'wb') as f: + for chunk in r: + f.write(chunk) + + dump = dict() + for k, v in r.headers.items(): + dump[k] = v + with open(http_headers_file, 'w') as fp: + json.dump(dump, fp, indent=2, sort_keys=True) + new_files = True + + if new_files: + os.chdir(tempdirbase) + make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog', + git_remote=options.git_remote) + shutil.rmtree(tempdirbase, ignore_errors=True) + +if __name__ == "__main__": + main() diff --git a/fdroidserver/index.py b/fdroidserver/index.py index b42c63d1..e029fc50 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -220,7 +220,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): json_name = 'index-v1.json' index_file = os.path.join(repodir, json_name) with open(index_file, 'w') as fp: - json.dump(output, fp, default=_index_encoder_default) + if common.options.pretty: + json.dump(output, fp, default=_index_encoder_default, indent=2) + else: + json.dump(output, fp, default=_index_encoder_default) if common.options.nosign: logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0e42d684..e26388b6 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -23,13 +23,11 @@ import sys import os import shutil import glob -import json import re import socket import zipfile import hashlib import pickle -import platform from datetime import datetime, timedelta from argparse import ArgumentParser @@ -39,6 +37,7 @@ from binascii import hexlify from PIL import Image import logging +from . import btlog from . import common from . import index from . import metadata @@ -1212,80 +1211,6 @@ def add_apks_to_per_app_repos(repodir, apks): shutil.copy(apkascpath, apk['per_app_repo']) -def make_binary_transparency_log(repodirs): - '''Log the indexes in a standalone git repo to serve as a "binary - transparency" log. - - see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies - - ''' - - import git - btrepo = 'binary_transparency' - if os.path.exists(os.path.join(btrepo, '.git')): - gitrepo = git.Repo(btrepo) - else: - if not os.path.exists(btrepo): - os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo) - - gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', 'fdroid update') - gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - - url = config['repo_url'].rstrip('/') - with open(os.path.join(btrepo, 'README.md'), 'w') as fp: - fp.write(""" -# Binary Transparency Log for %s - -""" % url[:url.rindex('/')]) # strip '/repo' - gitrepo.index.add(['README.md', ]) - gitrepo.index.commit('add README') - - for repodir in repodirs: - cpdir = os.path.join(btrepo, repodir) - if not os.path.exists(cpdir): - os.mkdir(cpdir) - for f in ('index.xml', 'index-v1.json'): - dest = os.path.join(cpdir, f) - shutil.copyfile(os.path.join(repodir, f), dest) - gitrepo.index.add([os.path.join(repodir, f), ]) - for f in ('index.jar', 'index-v1.jar'): - repof = os.path.join(repodir, f) - dest = os.path.join(cpdir, f) - jarin = zipfile.ZipFile(repof, 'r') - jarout = zipfile.ZipFile(dest, 'w') - for info in jarin.infolist(): - if info.filename.startswith('META-INF/'): - jarout.writestr(info, jarin.read(info.filename)) - jarout.close() - jarin.close() - gitrepo.index.add([repof, ]) - - files = [] - for root, dirs, filenames in os.walk(repodir): - for f in filenames: - files.append(os.path.relpath(os.path.join(root, f), repodir)) - output = collections.OrderedDict() - for f in sorted(files): - repofile = os.path.join(repodir, f) - stat = os.stat(repofile) - output[f] = ( - stat.st_size, - stat.st_ctime_ns, - stat.st_mtime_ns, - stat.st_mode, - stat.st_uid, - stat.st_gid, - ) - fslogfile = os.path.join(cpdir, 'filesystemlog.json') - with open(fslogfile, 'w') as fp: - json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - - gitrepo.index.commit('fdroid update') - - config = None options = None @@ -1483,8 +1408,9 @@ def main(): if len(repodirs) > 1: index.make(apps, sortedids, archapks, repodirs[1], True) - if config.get('binary_transparency_remote'): - make_binary_transparency_log(repodirs) + git_remote = config.get('binary_transparency_remote') + if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): + btlog.make_binary_transparency_log(repodirs, git_remote=git_remote) if config['update_stats']: # Update known apks info... diff --git a/tests/run-tests b/tests/run-tests index 743cc766..d0ccf083 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -592,12 +592,15 @@ echo_header "copy tests/repo, update with binary transparency log" REPOROOT=`create_test_dir` GNUPGHOME=$REPOROOT/gnupghome KEYSTORE=$WORKSPACE/tests/keystore.jks +mkdir $REPOROOT/git_remote +cd $REPOROOT/git_remote +git init --bare cd $REPOROOT $fdroid init --keystore $KEYSTORE --repo-keyalias=sova cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py -echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py +echo "binary_transparency_remote = '$REPOROOT/git_remote'" >> config.py echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py $fdroid update --verbose --pretty test -e repo/index.xml @@ -606,6 +609,8 @@ test -e repo/index-v1.jar grep -F '