From c591a4cd89d8d345fd6d163f496676f59db149b4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 27 Mar 2017 21:52:15 +0200 Subject: [PATCH] `fdroid btlog` external binary transparency logger This complements the binary transparency logging that happens on the server side !226. Anyone can set up an efficient tracker of any F-Droid repo which stores all index files that it sees. It uses HEAD requests and ETag checking to be as efficient as possible, so that this can be automatically run at a frequent pace. --- completion/bash-completion | 6 ++ fdroid | 1 + fdroidserver/btlog.py | 120 +++++++++++++++++++++++++++++++++++++ fdroidserver/common.py | 24 +++++--- 4 files changed, 144 insertions(+), 7 deletions(-) create mode 100755 fdroidserver/btlog.py diff --git a/completion/bash-completion b/completion/bash-completion index bdaa2cbb..48352447 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -236,6 +236,12 @@ __complete_verify() { esac } +__complete_btlog() { + opts="-u" + lopts="--git-remote --git-repo --url" + __complete_options +} + __complete_stats() { opts="-v -q -d" lopts="--verbose --quiet --download" diff --git a/fdroid b/fdroid index feea104a..bc1655b9 100755 --- a/fdroid +++ b/fdroid @@ -42,6 +42,7 @@ commands = { "stats": "Update the stats of the repo", "server": "Interact with the repo HTTP server", "signindex": "Sign indexes created using update --nosign", + "btlog": "Update the binary transparency log for a URL", } diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py new file mode 100755 index 00000000..1911a093 --- /dev/null +++ b/fdroidserver/btlog.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# btlog.py - part of the FDroid server tools +# Copyright (C) 2017, Hans-Christoph Steiner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# This is for creating a binary transparency log in a git repo for any +# F-Droid repo accessible via HTTP. It is meant to run very often, +# even once a minute in a cronjob, so it uses HEAD requests and the +# HTTP ETag to check if the file has changed. HEAD requests should +# not count against the download counts. This pattern of a HEAD then +# a GET is what fdroidclient uses to avoid ETags being abused as +# cookies. This also uses the same HTTP User Agent as the F-Droid +# client app so its not easy for the server to distinguish this from +# the F-Droid client. + +import os +import json +import logging +import requests +import shutil +import sys +import tempfile +from argparse import ArgumentParser + +from . import common + + +options = None + + +def main(): + global options + + parser = ArgumentParser(usage="%(prog)s [options]") + common.setup_global_opts(parser) + parser.add_argument("--git-repo", + default=os.path.join(os.getcwd(), 'binary_transparency'), + help="Path to the git repo to use as the log") + parser.add_argument("-u", "--url", default='https://f-droid.org', + help="The base URL for the repo to log (default: https://f-droid.org)") + parser.add_argument("--git-remote", default=None, + help="Create a repo signing key in a keystore") + options = parser.parse_args() + + if options.verbose: + logging.getLogger("requests").setLevel(logging.INFO) + logging.getLogger("urllib3").setLevel(logging.INFO) + else: + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + if not os.path.exists(options.git_repo): + logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo') + sys.exit(1) + + session = requests.Session() + + new_files = False + repodirs = ('repo', 'archive') + tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-') + for repodir in repodirs: + # TODO read HTTP headers for etag from git repo + tempdir = os.path.join(tempdirbase, repodir) + os.makedirs(tempdir, exist_ok=True) + gitrepodir = os.path.join(options.git_repo, repodir) + os.makedirs(gitrepodir, exist_ok=True) + for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'): + dlfile = os.path.join(tempdir, f) + dlurl = options.url + '/' + repodir + '/' + f + http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json') + + headers = { + 'User-Agent': 'F-Droid 0.102.3' + } + if os.path.exists(http_headers_file): + with open(http_headers_file) as fp: + etag = json.load(fp)['ETag'] + + r = session.head(dlurl, headers=headers, allow_redirects=False) + if r.status_code != 200: + logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl) + continue + if etag and etag == r.headers.get('ETag'): + logging.debug('ETag matches, did not download ' + dlurl) + continue + + r = session.get(dlurl, headers=headers, allow_redirects=False) + if r.status_code == 200: + with open(dlfile, 'wb') as f: + for chunk in r: + f.write(chunk) + + dump = dict() + for k, v in r.headers.items(): + dump[k] = v + with open(http_headers_file, 'w') as fp: + json.dump(dump, fp, indent=2, sort_keys=True) + new_files = True + + if new_files: + os.chdir(tempdirbase) + common.make_binary_transparency_log(repodirs, options.git_repo, options.url, + 'fdroid btlog') + shutil.rmtree(tempdirbase, ignore_errors=True) + +if __name__ == "__main__": + main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 00f22c7a..be902c47 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2353,7 +2353,9 @@ def is_repo_file(filename): ] -def make_binary_transparency_log(repodirs): +def make_binary_transparency_log(repodirs, btrepo='binary_transparency', + url=None, + commit_title='fdroid update'): '''Log the indexes in a standalone git repo to serve as a "binary transparency" log. @@ -2362,7 +2364,6 @@ def make_binary_transparency_log(repodirs): ''' import git - btrepo = 'binary_transparency' if os.path.exists(os.path.join(btrepo, '.git')): gitrepo = git.Repo(btrepo) else: @@ -2371,10 +2372,11 @@ def make_binary_transparency_log(repodirs): gitrepo = git.Repo.init(btrepo) gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', 'fdroid update') + gitconfig.set_value('user', 'name', commit_title) gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - url = config['repo_url'].rstrip('/') + if not url: + url = config['repo_url'].rstrip('/') with open(os.path.join(btrepo, 'README.md'), 'w') as fp: fp.write(""" # Binary Transparency Log for %s @@ -2388,11 +2390,16 @@ def make_binary_transparency_log(repodirs): if not os.path.exists(cpdir): os.mkdir(cpdir) for f in ('index.xml', 'index-v1.json'): + repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue dest = os.path.join(cpdir, f) - shutil.copyfile(os.path.join(repodir, f), dest) - gitrepo.index.add([os.path.join(repodir, f), ]) + shutil.copyfile(repof, dest) + gitrepo.index.add([repof, ]) for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) + if not os.path.exists(repof): + continue dest = os.path.join(cpdir, f) jarin = ZipFile(repof, 'r') jarout = ZipFile(dest, 'w') @@ -2424,4 +2431,7 @@ def make_binary_transparency_log(repodirs): json.dump(output, fp, indent=2) gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - gitrepo.index.commit('fdroid update') + for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): + gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) + + gitrepo.index.commit(commit_title)