From b7901952a18dcc1b387e6102e5ed6ded5760090c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Mar 2020 14:38:14 +0100 Subject: [PATCH] deploy: make androidobservatory and virustotal functions reusable This should not change the logic at all, just make the loop runs into standalone functions. --- .gitignore | 1 + fdroidserver/server.py | 232 ++++++++++++++++++++++------------------- tests/server.TestCase | 6 ++ 3 files changed, 134 insertions(+), 105 deletions(-) diff --git a/.gitignore b/.gitignore index 25aab3b3..57da9fb8 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ makebuildserver.config.py /tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png /tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk +/tests/virustotal/ /unsigned/ # generated by gettext diff --git a/fdroidserver/server.py b/fdroidserver/server.py index c01c1f26..57acd9ca 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -19,6 +19,7 @@ import sys import glob import hashlib +import json import os import paramiko import pwd @@ -447,9 +448,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): def upload_to_android_observatory(repo_section): - # depend on requests and lxml only if users enable AO import requests - from lxml.html import fromstring + requests # stop unused import warning if options.verbose: logging.getLogger("requests").setLevel(logging.INFO) @@ -460,44 +460,53 @@ def upload_to_android_observatory(repo_section): if repo_section == 'repo': for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))): - fpath = f - fname = os.path.basename(f) - r = requests.post('https://androidobservatory.org/', - data={'q': update.sha256sum(f), 'searchby': 'hash'}) - if r.status_code == 200: - # from now on XPath will be used to retrieve the message in the HTML - # androidobservatory doesn't have a nice API to talk with - # so we must scrape the page content - tree = fromstring(r.text) + upload_apk_to_android_observatory(f) - href = None - for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"): - a = element.attrib.get('href') - if a: - m = re.match(r'^/app/[0-9A-F]{40}$', a) - if m: - href = m.group() - page = 'https://androidobservatory.org' - message = '' - if href: - message = (_('Found {apkfilename} at {url}') - .format(apkfilename=fname, url=(page + href))) - if message: - logging.debug(message) - continue +def upload_apk_to_android_observatory(path): + # depend on requests and lxml only if users enable AO + import requests + from . import net + from lxml.html import fromstring - # upload the file with a post request - logging.info(_('Uploading {apkfilename} to androidobservatory.org') - .format(apkfilename=fname)) - r = requests.post('https://androidobservatory.org/upload', - files={'apk': (fname, open(fpath, 'rb'))}, - allow_redirects=False) + apkfilename = os.path.basename(path) + r = requests.post('https://androidobservatory.org/', + data={'q': update.sha256sum(path), 'searchby': 'hash'}, + headers=net.HEADERS) + if r.status_code == 200: + # from now on XPath will be used to retrieve the message in the HTML + # androidobservatory doesn't have a nice API to talk with + # so we must scrape the page content + tree = fromstring(r.text) + + href = None + for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"): + a = element.attrib.get('href') + if a: + m = re.match(r'^/app/[0-9A-F]{40}$', a) + if m: + href = m.group() + + page = 'https://androidobservatory.org' + message = '' + if href: + message = (_('Found {apkfilename} at {url}') + .format(apkfilename=apkfilename, url=(page + href))) + if message: + logging.debug(message) + + # upload the file with a post request + logging.info(_('Uploading {apkfilename} to androidobservatory.org') + .format(apkfilename=apkfilename)) + r = requests.post('https://androidobservatory.org/upload', + files={'apk': (apkfilename, open(path, 'rb'))}, + headers=net.HEADERS, + allow_redirects=False) def upload_to_virustotal(repo_section, virustotal_apikey): - import json import requests + requests # stop unused import warning logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) @@ -514,82 +523,95 @@ def upload_to_virustotal(repo_section, virustotal_apikey): for packageName, packages in data['packages'].items(): for package in packages: - outputfilename = os.path.join('virustotal', - packageName + '_' + str(package.get('versionCode')) - + '_' + package['hash'] + '.json') - if os.path.exists(outputfilename): - logging.debug(package['apkName'] + ' results are in ' + outputfilename) - continue - filename = package['apkName'] - repofilename = os.path.join(repo_section, filename) - logging.info('Checking if ' + repofilename + ' is on virustotal') + upload_apk_to_virustotal(virustotal_apikey, **package) - headers = { - "User-Agent": "F-Droid" - } - data = { - 'apikey': virustotal_apikey, - 'resource': package['hash'], - } - needs_file_upload = False - while True: - r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' - + urllib.parse.urlencode(data), headers=headers) - if r.status_code == 200: - response = r.json() - if response['response_code'] == 0: - needs_file_upload = True - else: - response['filename'] = filename - response['packageName'] = packageName - response['versionCode'] = package.get('versionCode') - response['versionName'] = package.get('versionName') - with open(outputfilename, 'w') as fp: - json.dump(response, fp, indent=2, sort_keys=True) - if response.get('positives', 0) > 0: - logging.warning(repofilename + ' has been flagged by virustotal ' - + str(response['positives']) + ' times:' - + '\n\t' + response['permalink']) - break - elif r.status_code == 204: - time.sleep(10) # wait for public API rate limiting +def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, + versionCode, **kwargs): + import requests - upload_url = None - if needs_file_upload: - manual_url = 'https://www.virustotal.com/' - size = os.path.getsize(repofilename) - if size > 200000000: - # VirusTotal API 200MB hard limit - logging.error(_('{path} more than 200MB, manually upload: {url}') - .format(path=repofilename, url=manual_url)) - elif size > 32000000: - # VirusTotal API requires fetching a URL to upload bigger files - r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' - + urllib.parse.urlencode(data), headers=headers) - if r.status_code == 200: - upload_url = r.json().get('upload_url') - elif r.status_code == 403: - logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' - + 'use {url} to upload {path}.') - .format(path=repofilename, url=manual_url)) - else: - r.raise_for_status() - else: - upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + outputfilename = os.path.join('virustotal', + packageName + '_' + str(versionCode) + + '_' + hash + '.json') + if os.path.exists(outputfilename): + logging.debug(apkName + ' results are in ' + outputfilename) + return outputfilename + repofilename = os.path.join('repo', apkName) + logging.info('Checking if ' + repofilename + ' is on virustotal') - if upload_url: - logging.info(_('Uploading {apkfilename} to virustotal') - .format(apkfilename=repofilename)) - files = { - 'file': (filename, open(repofilename, 'rb')) - } - r = requests.post(upload_url, data=data, headers=headers, files=files) - logging.debug(_('If this upload fails, try manually uploading to {url}') - .format(url=manual_url)) - r.raise_for_status() - response = r.json() - logging.info(response['verbose_msg'] + " " + response['permalink']) + headers = { + "User-Agent": "F-Droid" + } + if 'headers' in kwargs: + for k, v in kwargs['headers'].items(): + headers[k] = v + + data = { + 'apikey': virustotal_apikey, + 'resource': hash, + } + needs_file_upload = False + while True: + r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + response = r.json() + if response['response_code'] == 0: + needs_file_upload = True + else: + response['filename'] = apkName + response['packageName'] = packageName + response['versionCode'] = versionCode + if kwargs.get('versionName'): + response['versionName'] = kwargs.get('versionName') + with open(outputfilename, 'w') as fp: + json.dump(response, fp, indent=2, sort_keys=True) + + if response.get('positives', 0) > 0: + logging.warning(repofilename + ' has been flagged by virustotal ' + + str(response['positives']) + ' times:' + + '\n\t' + response['permalink']) + break + elif r.status_code == 204: + time.sleep(10) # wait for public API rate limiting + + upload_url = None + if needs_file_upload: + manual_url = 'https://www.virustotal.com/' + size = os.path.getsize(repofilename) + if size > 200000000: + # VirusTotal API 200MB hard limit + logging.error(_('{path} more than 200MB, manually upload: {url}') + .format(path=repofilename, url=manual_url)) + elif size > 32000000: + # VirusTotal API requires fetching a URL to upload bigger files + r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' + + urllib.parse.urlencode(data), headers=headers) + if r.status_code == 200: + upload_url = r.json().get('upload_url') + elif r.status_code == 403: + logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' + + 'use {url} to upload {path}.') + .format(path=repofilename, url=manual_url)) + else: + r.raise_for_status() + else: + upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' + + if upload_url: + logging.info(_('Uploading {apkfilename} to virustotal') + .format(apkfilename=repofilename)) + files = { + 'file': (apkName, open(repofilename, 'rb')) + } + r = requests.post(upload_url, data=data, headers=headers, files=files) + logging.debug(_('If this upload fails, try manually uploading to {url}') + .format(url=manual_url)) + r.raise_for_status() + response = r.json() + logging.info(response['verbose_msg'] + " " + response['permalink']) + + return outputfilename def push_binary_transparency(git_repo_path, git_remote): diff --git a/tests/server.TestCase b/tests/server.TestCase index 5d058153..8bd769ea 100755 --- a/tests/server.TestCase +++ b/tests/server.TestCase @@ -142,6 +142,12 @@ class ServerTest(unittest.TestCase): repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') + @unittest.skipIf(not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set') + def test_upload_to_virustotal(self): + fdroidserver.server.options.verbose = True + virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') + fdroidserver.server.upload_to_virustotal('repo', virustotal_apikey) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))