kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-14 11:00:10 +01:00
Code Activity

Merge branch 'morebintrans' into 'master'

Browse Source 
`fdroid btlog` for anyone to setup a binary transparency log for any repo

See merge request !243
This commit is contained in:
Torsten Grote 2017-04-05 12:45:40 +00:00
commit 4e4c6d9acf
6 changed files with 255 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
completion/bash-completion
View File

@ -236,6 +236,12 @@ __complete_verify() {
esac
}
__complete_btlog() {
opts="-u"
lopts="--git-remote --git-repo --url"
__complete_options
}
__complete_stats() {
opts="-v -q -d"
lopts="--verbose --quiet --download"

1
fdroid
View File

@ -42,6 +42,7 @@ commands = {
"stats": "Update the stats of the repo",
"server": "Interact with the repo HTTP server",
"signindex": "Sign indexes created using update --nosign",
"btlog": "Update the binary transparency log for a URL",
}

234
fdroidserver/btlog.py Executable file
View File

@ -0,0 +1,234 @@
#!/usr/bin/env python3
#
# btlog.py - part of the FDroid server tools
# Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# This is for creating a binary transparency log in a git repo for any
# F-Droid repo accessible via HTTP. It is meant to run very often,
# even once a minute in a cronjob, so it uses HEAD requests and the
# HTTP ETag to check if the file has changed. HEAD requests should
# not count against the download counts. This pattern of a HEAD then
# a GET is what fdroidclient uses to avoid ETags being abused as
# cookies. This also uses the same HTTP User Agent as the F-Droid
# client app so its not easy for the server to distinguish this from
# the F-Droid client.
import collections
import git
import glob
import os
import json
import logging
import requests
import shutil
import sys
import tempfile
import xml.dom.minidom
import zipfile
from argparse import ArgumentParser
from . import common
options = None
def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
url=None,
commit_title='fdroid update',
git_remote=None):
'''Log the indexes in a standalone git repo to serve as a "binary
transparency" log.
see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
'''
logging.info('Committing indexes to ' + btrepo)
if os.path.exists(os.path.join(btrepo, '.git')):
gitrepo = git.Repo(btrepo)
else:
if not os.path.exists(btrepo):
os.mkdir(btrepo)
gitrepo = git.Repo.init(btrepo)
if not url:
url = common.config['repo_url'].rstrip('/')
with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
fp.write("""
# Binary Transparency Log for %s
This is a log of the signed app index metadata. This is stored in a
git repo, which serves as an imperfect append-only storage mechanism.
People can then check that any file that they received from that
F-Droid repository was a publicly released file.
For more info on this idea:
* https://wiki.mozilla.org/Security/Binary_Transparency
""" % url[:url.rindex('/')]) # strip '/repo'
gitrepo.index.add(['README.md', ])
gitrepo.index.commit('add README')
for repodir in repodirs:
cpdir = os.path.join(btrepo, repodir)
if not os.path.exists(cpdir):
os.mkdir(cpdir)
for f in ('index.xml', 'index-v1.json'):
repof = os.path.join(repodir, f)
if not os.path.exists(repof):
continue
dest = os.path.join(cpdir, f)
if f.endswith('.xml'):
doc = xml.dom.minidom.parse(repof)
output = doc.toprettyxml(encoding='utf-8')
with open(dest, 'wb') as f:
f.write(output)
elif f.endswith('.json'):
with open(repof) as fp:
output = json.load(fp, object_pairs_hook=collections.OrderedDict)
with open(dest, 'w') as fp:
json.dump(output, fp, indent=2)
gitrepo.index.add([repof, ])
for f in ('index.jar', 'index-v1.jar'):
repof = os.path.join(repodir, f)
if not os.path.exists(repof):
continue
dest = os.path.join(cpdir, f)
jarin = zipfile.ZipFile(repof, 'r')
jarout = zipfile.ZipFile(dest, 'w')
for info in jarin.infolist():
if info.filename.startswith('META-INF/'):
jarout.writestr(info, jarin.read(info.filename))
jarout.close()
jarin.close()
gitrepo.index.add([repof, ])
files = []
for root, dirs, filenames in os.walk(repodir):
for f in filenames:
files.append(os.path.relpath(os.path.join(root, f), repodir))
output = collections.OrderedDict()
for f in sorted(files):
repofile = os.path.join(repodir, f)
stat = os.stat(repofile)
output[f] = (
stat.st_size,
stat.st_ctime_ns,
stat.st_mtime_ns,
stat.st_mode,
stat.st_uid,
stat.st_gid,
)
fslogfile = os.path.join(cpdir, 'filesystemlog.json')
with open(fslogfile, 'w') as fp:
json.dump(output, fp, indent=2)
gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
gitrepo.index.commit(commit_title)
if git_remote:
logging.info('Pushing binary transparency log to ' + git_remote)
origin = git.remote.Remote(gitrepo, 'origin')
if origin in gitrepo.remotes:
origin = gitrepo.remote('origin')
if 'set_url' in dir(origin): # added in GitPython 2.x
origin.set_url(git_remote)
else:
origin = gitrepo.create_remote('origin', git_remote)
origin.fetch()
origin.push('master')
def main():
global options
parser = ArgumentParser(usage="%(prog)s [options]")
common.setup_global_opts(parser)
parser.add_argument("--git-repo",
default=os.path.join(os.getcwd(), 'binary_transparency'),
help="Path to the git repo to use as the log")
parser.add_argument("-u", "--url", default='https://f-droid.org',
help="The base URL for the repo to log (default: https://f-droid.org)")
parser.add_argument("--git-remote", default=None,
help="Push the log to this git remote repository")
options = parser.parse_args()
if options.verbose:
logging.getLogger("requests").setLevel(logging.INFO)
logging.getLogger("urllib3").setLevel(logging.INFO)
else:
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
if not os.path.exists(options.git_repo):
logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo')
sys.exit(1)
session = requests.Session()
new_files = False
repodirs = ('repo', 'archive')
tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-')
for repodir in repodirs:
# TODO read HTTP headers for etag from git repo
tempdir = os.path.join(tempdirbase, repodir)
os.makedirs(tempdir, exist_ok=True)
gitrepodir = os.path.join(options.git_repo, repodir)
os.makedirs(gitrepodir, exist_ok=True)
for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'):
dlfile = os.path.join(tempdir, f)
dlurl = options.url + '/' + repodir + '/' + f
http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
headers = {
'User-Agent': 'F-Droid 0.102.3'
}
if os.path.exists(http_headers_file):
with open(http_headers_file) as fp:
etag = json.load(fp)['ETag']
r = session.head(dlurl, headers=headers, allow_redirects=False)
if r.status_code != 200:
logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl)
continue
if etag and etag == r.headers.get('ETag'):
logging.debug('ETag matches, did not download ' + dlurl)
continue
r = session.get(dlurl, headers=headers, allow_redirects=False)
if r.status_code == 200:
with open(dlfile, 'wb') as f:
for chunk in r:
f.write(chunk)
dump = dict()
for k, v in r.headers.items():
dump[k] = v
with open(http_headers_file, 'w') as fp:
json.dump(dump, fp, indent=2, sort_keys=True)
new_files = True
if new_files:
os.chdir(tempdirbase)
make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog',
git_remote=options.git_remote)
shutil.rmtree(tempdirbase, ignore_errors=True)
if __name__ == "__main__":
main()

5
fdroidserver/index.py
View File

@ -220,7 +220,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
json_name = 'index-v1.json'
index_file = os.path.join(repodir, json_name)
with open(index_file, 'w') as fp:
json.dump(output, fp, default=_index_encoder_default)
if common.options.pretty:
json.dump(output, fp, default=_index_encoder_default, indent=2)
else:
json.dump(output, fp, default=_index_encoder_default)
if common.options.nosign:
logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')

82
fdroidserver/update.py
View File

@ -23,13 +23,11 @@ import sys
import os
import shutil
import glob
import json
import re
import socket
import zipfile
import hashlib
import pickle
import platform
from datetime import datetime, timedelta
from argparse import ArgumentParser
@ -39,6 +37,7 @@ from binascii import hexlify
from PIL import Image
import logging
from . import btlog
from . import common
from . import index
from . import metadata
@ -1212,80 +1211,6 @@ def add_apks_to_per_app_repos(repodir, apks):
shutil.copy(apkascpath, apk['per_app_repo'])
def make_binary_transparency_log(repodirs):
'''Log the indexes in a standalone git repo to serve as a "binary
transparency" log.
see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
'''
import git
btrepo = 'binary_transparency'
if os.path.exists(os.path.join(btrepo, '.git')):
gitrepo = git.Repo(btrepo)
else:
if not os.path.exists(btrepo):
os.mkdir(btrepo)
gitrepo = git.Repo.init(btrepo)
gitconfig = gitrepo.config_writer()
gitconfig.set_value('user', 'name', 'fdroid update')
gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
url = config['repo_url'].rstrip('/')
with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
fp.write("""
# Binary Transparency Log for %s
""" % url[:url.rindex('/')]) # strip '/repo'
gitrepo.index.add(['README.md', ])
gitrepo.index.commit('add README')
for repodir in repodirs:
cpdir = os.path.join(btrepo, repodir)
if not os.path.exists(cpdir):
os.mkdir(cpdir)
for f in ('index.xml', 'index-v1.json'):
dest = os.path.join(cpdir, f)
shutil.copyfile(os.path.join(repodir, f), dest)
gitrepo.index.add([os.path.join(repodir, f), ])
for f in ('index.jar', 'index-v1.jar'):
repof = os.path.join(repodir, f)
dest = os.path.join(cpdir, f)
jarin = zipfile.ZipFile(repof, 'r')
jarout = zipfile.ZipFile(dest, 'w')
for info in jarin.infolist():
if info.filename.startswith('META-INF/'):
jarout.writestr(info, jarin.read(info.filename))
jarout.close()
jarin.close()
gitrepo.index.add([repof, ])
files = []
for root, dirs, filenames in os.walk(repodir):
for f in filenames:
files.append(os.path.relpath(os.path.join(root, f), repodir))
output = collections.OrderedDict()
for f in sorted(files):
repofile = os.path.join(repodir, f)
stat = os.stat(repofile)
output[f] = (
stat.st_size,
stat.st_ctime_ns,
stat.st_mtime_ns,
stat.st_mode,
stat.st_uid,
stat.st_gid,
)
fslogfile = os.path.join(cpdir, 'filesystemlog.json')
with open(fslogfile, 'w') as fp:
json.dump(output, fp, indent=2)
gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
gitrepo.index.commit('fdroid update')
config = None
options = None
@ -1483,8 +1408,9 @@ def main():
if len(repodirs) > 1:
index.make(apps, sortedids, archapks, repodirs[1], True)
if config.get('binary_transparency_remote'):
make_binary_transparency_log(repodirs)
git_remote = config.get('binary_transparency_remote')
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
btlog.make_binary_transparency_log(repodirs, git_remote=git_remote)
if config['update_stats']:
# Update known apks info...

7
tests/run-tests
View File

@ -592,12 +592,15 @@ echo_header "copy tests/repo, update with binary transparency log"
REPOROOT=`create_test_dir`
GNUPGHOME=$REPOROOT/gnupghome
KEYSTORE=$WORKSPACE/tests/keystore.jks
mkdir $REPOROOT/git_remote
cd $REPOROOT/git_remote
git init --bare
cd $REPOROOT
$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py
echo "binary_transparency_remote = '$REPOROOT/git_remote'" >> config.py
echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
$fdroid update --verbose --pretty
test -e repo/index.xml
@ -606,6 +609,8 @@ test -e repo/index-v1.jar
grep -F '<application id=' repo/index.xml > /dev/null
cd binary_transparency
[ `git rev-list --count HEAD` == "2" ]
cd $REPOROOT/git_remote
[ `git rev-list --count HEAD` == "2" ]
#------------------------------------------------------------------------------#