mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-19 19:40:17 +02:00
c9aa26d89e
Python encode/decode libs work directly with dicts, so the internal dict can just be passed directly to any of these libs (pyyaml, pyjson, msgpack, simplejson, etc). This still generates the exact same index.xml as before. This converts the internal format for the repo timestamp to a datetime instance, which can be easily converted to UNIX time in seconds for XML and UNIX time in milliseconds for the new index formats. UNIX time in milliseconds is directly serialized into a java.util.Date instance by Jackson.
445 lines
18 KiB
Python
445 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# server.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import sys
|
|
import glob
|
|
import hashlib
|
|
import os
|
|
import paramiko
|
|
import pwd
|
|
import subprocess
|
|
from argparse import ArgumentParser
|
|
import logging
|
|
import shutil
|
|
|
|
from . import common
|
|
|
|
config = None
|
|
options = None
|
|
|
|
|
|
def update_awsbucket(repo_section):
|
|
'''
|
|
Upload the contents of the directory `repo_section` (including
|
|
subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
|
|
bucket will first be deleted.
|
|
|
|
Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
|
|
'''
|
|
|
|
logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
|
|
+ config['awsbucket'] + '"')
|
|
|
|
import libcloud.security
|
|
libcloud.security.VERIFY_SSL_CERT = True
|
|
from libcloud.storage.types import Provider, ContainerDoesNotExistError
|
|
from libcloud.storage.providers import get_driver
|
|
|
|
if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
|
|
logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
|
|
sys.exit(1)
|
|
awsbucket = config['awsbucket']
|
|
|
|
cls = get_driver(Provider.S3)
|
|
driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
|
|
try:
|
|
container = driver.get_container(container_name=awsbucket)
|
|
except ContainerDoesNotExistError:
|
|
container = driver.create_container(container_name=awsbucket)
|
|
logging.info('Created new container "' + container.name + '"')
|
|
|
|
upload_dir = 'fdroid/' + repo_section
|
|
objs = dict()
|
|
for obj in container.list_objects():
|
|
if obj.name.startswith(upload_dir + '/'):
|
|
objs[obj.name] = obj
|
|
|
|
for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
|
|
for name in files:
|
|
upload = False
|
|
file_to_upload = os.path.join(root, name)
|
|
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
|
if object_name not in objs:
|
|
upload = True
|
|
else:
|
|
obj = objs.pop(object_name)
|
|
if obj.size != os.path.getsize(file_to_upload):
|
|
upload = True
|
|
else:
|
|
# if the sizes match, then compare by MD5
|
|
md5 = hashlib.md5()
|
|
with open(file_to_upload, 'rb') as f:
|
|
while True:
|
|
data = f.read(8192)
|
|
if not data:
|
|
break
|
|
md5.update(data)
|
|
if obj.hash != md5.hexdigest():
|
|
s3url = 's3://' + awsbucket + '/' + obj.name
|
|
logging.info(' deleting ' + s3url)
|
|
if not driver.delete_object(obj):
|
|
logging.warn('Could not delete ' + s3url)
|
|
upload = True
|
|
|
|
if upload:
|
|
logging.debug(' uploading "' + file_to_upload + '"...')
|
|
extra = {'acl': 'public-read'}
|
|
if file_to_upload.endswith('.sig'):
|
|
extra['content_type'] = 'application/pgp-signature'
|
|
elif file_to_upload.endswith('.asc'):
|
|
extra['content_type'] = 'application/pgp-signature'
|
|
logging.info(' uploading ' + os.path.relpath(file_to_upload)
|
|
+ ' to s3://' + awsbucket + '/' + object_name)
|
|
with open(file_to_upload, 'rb') as iterator:
|
|
obj = driver.upload_object_via_stream(iterator=iterator,
|
|
container=container,
|
|
object_name=object_name,
|
|
extra=extra)
|
|
# delete the remnants in the bucket, they do not exist locally
|
|
while objs:
|
|
object_name, obj = objs.popitem()
|
|
s3url = 's3://' + awsbucket + '/' + object_name
|
|
if object_name.startswith(upload_dir):
|
|
logging.warn(' deleting ' + s3url)
|
|
driver.delete_object(obj)
|
|
else:
|
|
logging.info(' skipping ' + s3url)
|
|
|
|
|
|
def update_serverwebroot(serverwebroot, repo_section):
|
|
# use a checksum comparison for accurate comparisons on different
|
|
# filesystems, for example, FAT has a low resolution timestamp
|
|
rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
|
|
if not options.no_checksum:
|
|
rsyncargs.append('--checksum')
|
|
if options.verbose:
|
|
rsyncargs += ['--verbose']
|
|
if options.quiet:
|
|
rsyncargs += ['--quiet']
|
|
if options.identity_file is not None:
|
|
rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
|
|
if 'identity_file' in config:
|
|
rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
|
|
indexxml = os.path.join(repo_section, 'index.xml')
|
|
indexjar = os.path.join(repo_section, 'index.jar')
|
|
indexv1jar = os.path.join(repo_section, 'index-v1.jar')
|
|
# Upload the first time without the index files and delay the deletion as
|
|
# much as possible, that keeps the repo functional while this update is
|
|
# running. Then once it is complete, rerun the command again to upload
|
|
# the index files. Always using the same target with rsync allows for
|
|
# very strict settings on the receiving server, you can literally specify
|
|
# the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
|
|
# (serverwebroot is guaranteed to have a trailing slash in common.py)
|
|
logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
|
|
if subprocess.call(rsyncargs +
|
|
['--exclude', indexxml, '--exclude', indexjar,
|
|
'--exclude', indexv1jar,
|
|
repo_section, serverwebroot]) != 0:
|
|
sys.exit(1)
|
|
if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
|
|
sys.exit(1)
|
|
# upload "current version" symlinks if requested
|
|
if config['make_current_version_link'] and repo_section == 'repo':
|
|
links_to_upload = []
|
|
for f in glob.glob('*.apk') \
|
|
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
|
if os.path.islink(f):
|
|
links_to_upload.append(f)
|
|
if len(links_to_upload) > 0:
|
|
if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
|
|
sys.exit(1)
|
|
|
|
|
|
def _local_sync(fromdir, todir):
|
|
rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
|
|
'--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
|
|
# use stricter rsync checking on all files since people using offline mode
|
|
# are already prioritizing security above ease and speed
|
|
if not options.no_checksum:
|
|
rsyncargs.append('--checksum')
|
|
if options.verbose:
|
|
rsyncargs += ['--verbose']
|
|
if options.quiet:
|
|
rsyncargs += ['--quiet']
|
|
logging.debug(' '.join(rsyncargs + [fromdir, todir]))
|
|
if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
|
|
sys.exit(1)
|
|
|
|
|
|
def sync_from_localcopy(repo_section, local_copy_dir):
|
|
logging.info('Syncing from local_copy_dir to this repo.')
|
|
# trailing slashes have a meaning in rsync which is not needed here, so
|
|
# make sure both paths have exactly one trailing slash
|
|
_local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
|
|
repo_section.rstrip('/') + '/')
|
|
|
|
|
|
def update_localcopy(repo_section, local_copy_dir):
|
|
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
|
_local_sync(repo_section, local_copy_dir)
|
|
|
|
|
|
def update_servergitmirrors(servergitmirrors, repo_section):
|
|
# depend on GitPython only if users set a git mirror
|
|
import git
|
|
# right now we support only 'repo' git-mirroring
|
|
if repo_section == 'repo':
|
|
# create a new git-mirror folder
|
|
repo_dir = os.path.join('.', 'git-mirror/')
|
|
|
|
# remove if already present
|
|
if os.path.isdir(repo_dir):
|
|
shutil.rmtree(repo_dir)
|
|
|
|
repo = git.Repo.init(repo_dir)
|
|
|
|
# take care of each mirror
|
|
for mirror in servergitmirrors:
|
|
hostname = mirror.split("/")[2]
|
|
repo.create_remote(hostname, mirror)
|
|
logging.info('Mirroring to: ' + mirror)
|
|
|
|
# copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync
|
|
fdroid_repo_path = os.path.join(repo_dir, "fdroid")
|
|
_local_sync(repo_section, fdroid_repo_path)
|
|
|
|
# sadly index.add don't allow the --all parameter
|
|
repo.git.add(all=True)
|
|
repo.index.commit("fdroidserver git-mirror")
|
|
|
|
# push for every remote. This will overwrite the git history
|
|
for remote in repo.remotes:
|
|
remote.push('master', force=True, set_upstream=True)
|
|
|
|
|
|
def upload_to_android_observatory(repo_section):
|
|
# depend on requests and lxml only if users enable AO
|
|
import requests
|
|
from lxml.html import fromstring
|
|
|
|
if repo_section == 'repo':
|
|
for f in glob.glob(os.path.join(repo_section, '*.apk')):
|
|
fpath = f
|
|
fname = os.path.basename(f)
|
|
logging.info('Uploading ' + fname + ' to androidobservatory.org')
|
|
|
|
# upload the file with a post request
|
|
r = requests.post('https://androidobservatory.org/upload', files={'apk': (fname, open(fpath, 'rb'))})
|
|
response = r.text
|
|
page = r.url
|
|
|
|
# from now on XPath will be used to retrieve the message in the HTML
|
|
# androidobservatory doesn't have a nice API to talk with
|
|
# so we must scrape the page content
|
|
tree = fromstring(response)
|
|
alert = tree.xpath("//html/body/div[@class='container content-container']/div[@class='alert alert-info']")[0]
|
|
|
|
message = ""
|
|
appurl = page
|
|
for el in alert:
|
|
# if the application was added successfully we retrive the url
|
|
# if the application was already uploaded we use the redirect page url
|
|
if el.attrib.get("href") is not None:
|
|
appurl = page + el.attrib["href"][1:]
|
|
message += el.text.replace(" here", "") + el.tail
|
|
else:
|
|
message += el.tail
|
|
message = message.strip() + " " + appurl
|
|
logging.info(message)
|
|
|
|
|
|
def upload_to_virustotal(repo_section, vt_apikey):
|
|
import requests
|
|
|
|
if repo_section == 'repo':
|
|
for f in glob.glob(os.path.join(repo_section, '*.apk')):
|
|
fpath = f
|
|
fname = os.path.basename(f)
|
|
logging.info('Uploading ' + fname + ' to virustotal.com')
|
|
|
|
# upload the file with a post request
|
|
params = {'apikey': vt_apikey}
|
|
files = {'file': (fname, open(fpath, 'rb'))}
|
|
r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=params)
|
|
response = r.json()
|
|
|
|
logging.info(response['verbose_msg'] + " " + response['permalink'])
|
|
|
|
|
|
def push_binary_transparency(binary_transparency_remote):
|
|
'''push the binary transparency git repo to the specifed remote'''
|
|
import git
|
|
|
|
repo = git.Repo('binary_transparency_log')
|
|
pushremote = None
|
|
for remote in repo.remotes:
|
|
if remote.url == binary_transparency_remote:
|
|
pushremote = remote
|
|
break
|
|
|
|
if not pushremote:
|
|
pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote)
|
|
pushremote.push('master')
|
|
|
|
|
|
def main():
|
|
global config, options
|
|
|
|
# Parse command line...
|
|
parser = ArgumentParser()
|
|
common.setup_global_opts(parser)
|
|
parser.add_argument("command", help="command to execute, either 'init' or 'update'")
|
|
parser.add_argument("-i", "--identity-file", default=None,
|
|
help="Specify an identity file to provide to SSH for rsyncing")
|
|
parser.add_argument("--local-copy-dir", default=None,
|
|
help="Specify a local folder to sync the repo to")
|
|
parser.add_argument("--sync-from-local-copy-dir", action="store_true", default=False,
|
|
help="Before uploading to servers, sync from local copy dir")
|
|
parser.add_argument("--no-checksum", action="store_true", default=False,
|
|
help="Don't use rsync checksums")
|
|
options = parser.parse_args()
|
|
|
|
config = common.read_config(options)
|
|
|
|
if options.command != 'init' and options.command != 'update':
|
|
logging.critical("The only commands currently supported are 'init' and 'update'")
|
|
sys.exit(1)
|
|
|
|
if config.get('nonstandardwebroot') is True:
|
|
standardwebroot = False
|
|
else:
|
|
standardwebroot = True
|
|
|
|
for serverwebroot in config.get('serverwebroot', []):
|
|
# this supports both an ssh host:path and just a path
|
|
s = serverwebroot.rstrip('/').split(':')
|
|
if len(s) == 1:
|
|
fdroiddir = s[0]
|
|
elif len(s) == 2:
|
|
host, fdroiddir = s
|
|
else:
|
|
logging.error('Malformed serverwebroot line: ' + serverwebroot)
|
|
sys.exit(1)
|
|
repobase = os.path.basename(fdroiddir)
|
|
if standardwebroot and repobase != 'fdroid':
|
|
logging.error('serverwebroot path does not end with "fdroid", '
|
|
+ 'perhaps you meant one of these:\n\t'
|
|
+ serverwebroot.rstrip('/') + '/fdroid\n\t'
|
|
+ serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
|
|
sys.exit(1)
|
|
|
|
if options.local_copy_dir is not None:
|
|
local_copy_dir = options.local_copy_dir
|
|
elif config.get('local_copy_dir'):
|
|
local_copy_dir = config['local_copy_dir']
|
|
else:
|
|
local_copy_dir = None
|
|
if local_copy_dir is not None:
|
|
fdroiddir = local_copy_dir.rstrip('/')
|
|
if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
|
|
logging.error('local_copy_dir must be directory, not a file!')
|
|
sys.exit(1)
|
|
if not os.path.exists(os.path.dirname(fdroiddir)):
|
|
logging.error('The root dir for local_copy_dir "'
|
|
+ os.path.dirname(fdroiddir)
|
|
+ '" does not exist!')
|
|
sys.exit(1)
|
|
if not os.path.isabs(fdroiddir):
|
|
logging.error('local_copy_dir must be an absolute path!')
|
|
sys.exit(1)
|
|
repobase = os.path.basename(fdroiddir)
|
|
if standardwebroot and repobase != 'fdroid':
|
|
logging.error('local_copy_dir does not end with "fdroid", '
|
|
+ 'perhaps you meant: ' + fdroiddir + '/fdroid')
|
|
sys.exit(1)
|
|
if local_copy_dir[-1] != '/':
|
|
local_copy_dir += '/'
|
|
local_copy_dir = local_copy_dir.replace('//', '/')
|
|
if not os.path.exists(fdroiddir):
|
|
os.mkdir(fdroiddir)
|
|
|
|
if not config.get('awsbucket') \
|
|
and not config.get('serverwebroot') \
|
|
and not config.get('servergitmirrors') \
|
|
and not config.get('uploadto_androidobservatory') \
|
|
and not config.get('virustotal_apikey') \
|
|
and local_copy_dir is None:
|
|
logging.warn('No option set! Edit your config.py to set at least one among:\n'
|
|
+ 'serverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey or uploadto_androidobservatory')
|
|
sys.exit(1)
|
|
|
|
repo_sections = ['repo']
|
|
if config['archive_older'] != 0:
|
|
repo_sections.append('archive')
|
|
if not os.path.exists('archive'):
|
|
os.mkdir('archive')
|
|
if config['per_app_repos']:
|
|
repo_sections += common.get_per_app_repos()
|
|
|
|
if options.command == 'init':
|
|
ssh = paramiko.SSHClient()
|
|
ssh.load_system_host_keys()
|
|
for serverwebroot in config.get('serverwebroot', []):
|
|
sshstr, remotepath = serverwebroot.rstrip('/').split(':')
|
|
if sshstr.find('@') >= 0:
|
|
username, hostname = sshstr.split('@')
|
|
else:
|
|
username = pwd.getpwuid(os.getuid())[0] # get effective uid
|
|
hostname = sshstr
|
|
ssh.connect(hostname, username=username)
|
|
sftp = ssh.open_sftp()
|
|
if os.path.basename(remotepath) \
|
|
not in sftp.listdir(os.path.dirname(remotepath)):
|
|
sftp.mkdir(remotepath, mode=0o755)
|
|
for repo_section in repo_sections:
|
|
repo_path = os.path.join(remotepath, repo_section)
|
|
if os.path.basename(repo_path) \
|
|
not in sftp.listdir(remotepath):
|
|
sftp.mkdir(repo_path, mode=0o755)
|
|
sftp.close()
|
|
ssh.close()
|
|
elif options.command == 'update':
|
|
for repo_section in repo_sections:
|
|
if local_copy_dir is not None:
|
|
if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
|
|
sync_from_localcopy(repo_section, local_copy_dir)
|
|
else:
|
|
update_localcopy(repo_section, local_copy_dir)
|
|
for serverwebroot in config.get('serverwebroot', []):
|
|
update_serverwebroot(serverwebroot, repo_section)
|
|
if config.get('servergitmirrors', []):
|
|
# update_servergitmirrors will take care of multiple mirrors so don't need a foreach
|
|
servergitmirrors = config.get('servergitmirrors', [])
|
|
update_servergitmirrors(servergitmirrors, repo_section)
|
|
if config.get('awsbucket'):
|
|
update_awsbucket(repo_section)
|
|
if config.get('uploadto_androidobservatory'):
|
|
upload_to_android_observatory(repo_section)
|
|
if config.get('virustotal_apikey'):
|
|
upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
|
|
|
|
binary_transparency_remote = config.get('binary_transparency_remote')
|
|
if binary_transparency_remote:
|
|
push_binary_transparency(binary_transparency_remote)
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|