diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0c12c3f9..00f22c7a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -20,12 +20,15 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import collections import io import os import sys import re import shutil import glob +import json +import platform import stat import subprocess import time @@ -2348,3 +2351,77 @@ def is_repo_file(filename): b'index-v1.json', b'categories.txt', ] + + +def make_binary_transparency_log(repodirs): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + + import git + btrepo = 'binary_transparency' + if os.path.exists(os.path.join(btrepo, '.git')): + gitrepo = git.Repo(btrepo) + else: + if not os.path.exists(btrepo): + os.mkdir(btrepo) + gitrepo = git.Repo.init(btrepo) + + gitconfig = gitrepo.config_writer() + gitconfig.set_value('user', 'name', 'fdroid update') + gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) + + url = config['repo_url'].rstrip('/') + with open(os.path.join(btrepo, 'README.md'), 'w') as fp: + fp.write(""" +# Binary Transparency Log for %s + +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) + gitrepo.index.commit('add README') + + for repodir in repodirs: + cpdir = os.path.join(btrepo, repodir) + if not os.path.exists(cpdir): + os.mkdir(cpdir) + for f in ('index.xml', 'index-v1.json'): + dest = os.path.join(cpdir, f) + shutil.copyfile(os.path.join(repodir, f), dest) + gitrepo.index.add([os.path.join(repodir, f), ]) + for f in ('index.jar', 'index-v1.jar'): + repof = os.path.join(repodir, f) + dest = os.path.join(cpdir, f) + jarin = ZipFile(repof, 'r') + jarout = ZipFile(dest, 'w') + for info in jarin.infolist(): + if info.filename.startswith('META-INF/'): + jarout.writestr(info, jarin.read(info.filename)) + jarout.close() + jarin.close() + gitrepo.index.add([repof, ]) + + files = [] + for root, dirs, filenames in os.walk(repodir): + for f in filenames: + files.append(os.path.relpath(os.path.join(root, f), repodir)) + output = collections.OrderedDict() + for f in sorted(files): + repofile = os.path.join(repodir, f) + stat = os.stat(repofile) + output[f] = ( + stat.st_size, + stat.st_ctime_ns, + stat.st_mtime_ns, + stat.st_mode, + stat.st_uid, + stat.st_gid, + ) + fslogfile = os.path.join(cpdir, 'filesystemlog.json') + with open(fslogfile, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + + gitrepo.index.commit('fdroid update') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0e42d684..943d3b8e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -23,13 +23,11 @@ import sys import os import shutil import glob -import json import re import socket import zipfile import hashlib import pickle -import platform from datetime import datetime, timedelta from argparse import ArgumentParser @@ -1212,80 +1210,6 @@ def add_apks_to_per_app_repos(repodir, apks): shutil.copy(apkascpath, apk['per_app_repo']) -def make_binary_transparency_log(repodirs): - '''Log the indexes in a standalone git repo to serve as a "binary - transparency" log. - - see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies - - ''' - - import git - btrepo = 'binary_transparency' - if os.path.exists(os.path.join(btrepo, '.git')): - gitrepo = git.Repo(btrepo) - else: - if not os.path.exists(btrepo): - os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo) - - gitconfig = gitrepo.config_writer() - gitconfig.set_value('user', 'name', 'fdroid update') - gitconfig.set_value('user', 'email', 'fdroid@' + platform.node()) - - url = config['repo_url'].rstrip('/') - with open(os.path.join(btrepo, 'README.md'), 'w') as fp: - fp.write(""" -# Binary Transparency Log for %s - -""" % url[:url.rindex('/')]) # strip '/repo' - gitrepo.index.add(['README.md', ]) - gitrepo.index.commit('add README') - - for repodir in repodirs: - cpdir = os.path.join(btrepo, repodir) - if not os.path.exists(cpdir): - os.mkdir(cpdir) - for f in ('index.xml', 'index-v1.json'): - dest = os.path.join(cpdir, f) - shutil.copyfile(os.path.join(repodir, f), dest) - gitrepo.index.add([os.path.join(repodir, f), ]) - for f in ('index.jar', 'index-v1.jar'): - repof = os.path.join(repodir, f) - dest = os.path.join(cpdir, f) - jarin = zipfile.ZipFile(repof, 'r') - jarout = zipfile.ZipFile(dest, 'w') - for info in jarin.infolist(): - if info.filename.startswith('META-INF/'): - jarout.writestr(info, jarin.read(info.filename)) - jarout.close() - jarin.close() - gitrepo.index.add([repof, ]) - - files = [] - for root, dirs, filenames in os.walk(repodir): - for f in filenames: - files.append(os.path.relpath(os.path.join(root, f), repodir)) - output = collections.OrderedDict() - for f in sorted(files): - repofile = os.path.join(repodir, f) - stat = os.stat(repofile) - output[f] = ( - stat.st_size, - stat.st_ctime_ns, - stat.st_mtime_ns, - stat.st_mode, - stat.st_uid, - stat.st_gid, - ) - fslogfile = os.path.join(cpdir, 'filesystemlog.json') - with open(fslogfile, 'w') as fp: - json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) - - gitrepo.index.commit('fdroid update') - - config = None options = None @@ -1484,7 +1408,7 @@ def main(): index.make(apps, sortedids, archapks, repodirs[1], True) if config.get('binary_transparency_remote'): - make_binary_transparency_log(repodirs) + common.make_binary_transparency_log(repodirs) if config['update_stats']: # Update known apks info...