mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
deploy: make androidobservatory and virustotal functions reusable
This should not change the logic at all, just make the loop runs into standalone functions.
This commit is contained in:
parent
733e7be1b3
commit
b7901952a1
1
.gitignore
vendored
1
.gitignore
vendored
@ -57,6 +57,7 @@ makebuildserver.config.py
|
|||||||
/tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png
|
/tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png
|
||||||
/tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png
|
/tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png
|
||||||
/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
|
/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
|
||||||
|
/tests/virustotal/
|
||||||
/unsigned/
|
/unsigned/
|
||||||
|
|
||||||
# generated by gettext
|
# generated by gettext
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
import sys
|
import sys
|
||||||
import glob
|
import glob
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import paramiko
|
import paramiko
|
||||||
import pwd
|
import pwd
|
||||||
@ -447,9 +448,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||||||
|
|
||||||
|
|
||||||
def upload_to_android_observatory(repo_section):
|
def upload_to_android_observatory(repo_section):
|
||||||
# depend on requests and lxml only if users enable AO
|
|
||||||
import requests
|
import requests
|
||||||
from lxml.html import fromstring
|
requests # stop unused import warning
|
||||||
|
|
||||||
if options.verbose:
|
if options.verbose:
|
||||||
logging.getLogger("requests").setLevel(logging.INFO)
|
logging.getLogger("requests").setLevel(logging.INFO)
|
||||||
@ -460,44 +460,53 @@ def upload_to_android_observatory(repo_section):
|
|||||||
|
|
||||||
if repo_section == 'repo':
|
if repo_section == 'repo':
|
||||||
for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))):
|
for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))):
|
||||||
fpath = f
|
upload_apk_to_android_observatory(f)
|
||||||
fname = os.path.basename(f)
|
|
||||||
r = requests.post('https://androidobservatory.org/',
|
|
||||||
data={'q': update.sha256sum(f), 'searchby': 'hash'})
|
|
||||||
if r.status_code == 200:
|
|
||||||
# from now on XPath will be used to retrieve the message in the HTML
|
|
||||||
# androidobservatory doesn't have a nice API to talk with
|
|
||||||
# so we must scrape the page content
|
|
||||||
tree = fromstring(r.text)
|
|
||||||
|
|
||||||
href = None
|
|
||||||
for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"):
|
|
||||||
a = element.attrib.get('href')
|
|
||||||
if a:
|
|
||||||
m = re.match(r'^/app/[0-9A-F]{40}$', a)
|
|
||||||
if m:
|
|
||||||
href = m.group()
|
|
||||||
|
|
||||||
page = 'https://androidobservatory.org'
|
def upload_apk_to_android_observatory(path):
|
||||||
message = ''
|
# depend on requests and lxml only if users enable AO
|
||||||
if href:
|
import requests
|
||||||
message = (_('Found {apkfilename} at {url}')
|
from . import net
|
||||||
.format(apkfilename=fname, url=(page + href)))
|
from lxml.html import fromstring
|
||||||
if message:
|
|
||||||
logging.debug(message)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# upload the file with a post request
|
apkfilename = os.path.basename(path)
|
||||||
logging.info(_('Uploading {apkfilename} to androidobservatory.org')
|
r = requests.post('https://androidobservatory.org/',
|
||||||
.format(apkfilename=fname))
|
data={'q': update.sha256sum(path), 'searchby': 'hash'},
|
||||||
r = requests.post('https://androidobservatory.org/upload',
|
headers=net.HEADERS)
|
||||||
files={'apk': (fname, open(fpath, 'rb'))},
|
if r.status_code == 200:
|
||||||
allow_redirects=False)
|
# from now on XPath will be used to retrieve the message in the HTML
|
||||||
|
# androidobservatory doesn't have a nice API to talk with
|
||||||
|
# so we must scrape the page content
|
||||||
|
tree = fromstring(r.text)
|
||||||
|
|
||||||
|
href = None
|
||||||
|
for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"):
|
||||||
|
a = element.attrib.get('href')
|
||||||
|
if a:
|
||||||
|
m = re.match(r'^/app/[0-9A-F]{40}$', a)
|
||||||
|
if m:
|
||||||
|
href = m.group()
|
||||||
|
|
||||||
|
page = 'https://androidobservatory.org'
|
||||||
|
message = ''
|
||||||
|
if href:
|
||||||
|
message = (_('Found {apkfilename} at {url}')
|
||||||
|
.format(apkfilename=apkfilename, url=(page + href)))
|
||||||
|
if message:
|
||||||
|
logging.debug(message)
|
||||||
|
|
||||||
|
# upload the file with a post request
|
||||||
|
logging.info(_('Uploading {apkfilename} to androidobservatory.org')
|
||||||
|
.format(apkfilename=apkfilename))
|
||||||
|
r = requests.post('https://androidobservatory.org/upload',
|
||||||
|
files={'apk': (apkfilename, open(path, 'rb'))},
|
||||||
|
headers=net.HEADERS,
|
||||||
|
allow_redirects=False)
|
||||||
|
|
||||||
|
|
||||||
def upload_to_virustotal(repo_section, virustotal_apikey):
|
def upload_to_virustotal(repo_section, virustotal_apikey):
|
||||||
import json
|
|
||||||
import requests
|
import requests
|
||||||
|
requests # stop unused import warning
|
||||||
|
|
||||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||||
@ -514,82 +523,95 @@ def upload_to_virustotal(repo_section, virustotal_apikey):
|
|||||||
|
|
||||||
for packageName, packages in data['packages'].items():
|
for packageName, packages in data['packages'].items():
|
||||||
for package in packages:
|
for package in packages:
|
||||||
outputfilename = os.path.join('virustotal',
|
upload_apk_to_virustotal(virustotal_apikey, **package)
|
||||||
packageName + '_' + str(package.get('versionCode'))
|
|
||||||
+ '_' + package['hash'] + '.json')
|
|
||||||
if os.path.exists(outputfilename):
|
|
||||||
logging.debug(package['apkName'] + ' results are in ' + outputfilename)
|
|
||||||
continue
|
|
||||||
filename = package['apkName']
|
|
||||||
repofilename = os.path.join(repo_section, filename)
|
|
||||||
logging.info('Checking if ' + repofilename + ' is on virustotal')
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
"User-Agent": "F-Droid"
|
|
||||||
}
|
|
||||||
data = {
|
|
||||||
'apikey': virustotal_apikey,
|
|
||||||
'resource': package['hash'],
|
|
||||||
}
|
|
||||||
needs_file_upload = False
|
|
||||||
while True:
|
|
||||||
r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?'
|
|
||||||
+ urllib.parse.urlencode(data), headers=headers)
|
|
||||||
if r.status_code == 200:
|
|
||||||
response = r.json()
|
|
||||||
if response['response_code'] == 0:
|
|
||||||
needs_file_upload = True
|
|
||||||
else:
|
|
||||||
response['filename'] = filename
|
|
||||||
response['packageName'] = packageName
|
|
||||||
response['versionCode'] = package.get('versionCode')
|
|
||||||
response['versionName'] = package.get('versionName')
|
|
||||||
with open(outputfilename, 'w') as fp:
|
|
||||||
json.dump(response, fp, indent=2, sort_keys=True)
|
|
||||||
|
|
||||||
if response.get('positives', 0) > 0:
|
def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash,
|
||||||
logging.warning(repofilename + ' has been flagged by virustotal '
|
versionCode, **kwargs):
|
||||||
+ str(response['positives']) + ' times:'
|
import requests
|
||||||
+ '\n\t' + response['permalink'])
|
|
||||||
break
|
|
||||||
elif r.status_code == 204:
|
|
||||||
time.sleep(10) # wait for public API rate limiting
|
|
||||||
|
|
||||||
upload_url = None
|
outputfilename = os.path.join('virustotal',
|
||||||
if needs_file_upload:
|
packageName + '_' + str(versionCode)
|
||||||
manual_url = 'https://www.virustotal.com/'
|
+ '_' + hash + '.json')
|
||||||
size = os.path.getsize(repofilename)
|
if os.path.exists(outputfilename):
|
||||||
if size > 200000000:
|
logging.debug(apkName + ' results are in ' + outputfilename)
|
||||||
# VirusTotal API 200MB hard limit
|
return outputfilename
|
||||||
logging.error(_('{path} more than 200MB, manually upload: {url}')
|
repofilename = os.path.join('repo', apkName)
|
||||||
.format(path=repofilename, url=manual_url))
|
logging.info('Checking if ' + repofilename + ' is on virustotal')
|
||||||
elif size > 32000000:
|
|
||||||
# VirusTotal API requires fetching a URL to upload bigger files
|
|
||||||
r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?'
|
|
||||||
+ urllib.parse.urlencode(data), headers=headers)
|
|
||||||
if r.status_code == 200:
|
|
||||||
upload_url = r.json().get('upload_url')
|
|
||||||
elif r.status_code == 403:
|
|
||||||
logging.error(_('VirusTotal API key cannot upload files larger than 32MB, '
|
|
||||||
+ 'use {url} to upload {path}.')
|
|
||||||
.format(path=repofilename, url=manual_url))
|
|
||||||
else:
|
|
||||||
r.raise_for_status()
|
|
||||||
else:
|
|
||||||
upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan'
|
|
||||||
|
|
||||||
if upload_url:
|
headers = {
|
||||||
logging.info(_('Uploading {apkfilename} to virustotal')
|
"User-Agent": "F-Droid"
|
||||||
.format(apkfilename=repofilename))
|
}
|
||||||
files = {
|
if 'headers' in kwargs:
|
||||||
'file': (filename, open(repofilename, 'rb'))
|
for k, v in kwargs['headers'].items():
|
||||||
}
|
headers[k] = v
|
||||||
r = requests.post(upload_url, data=data, headers=headers, files=files)
|
|
||||||
logging.debug(_('If this upload fails, try manually uploading to {url}')
|
data = {
|
||||||
.format(url=manual_url))
|
'apikey': virustotal_apikey,
|
||||||
r.raise_for_status()
|
'resource': hash,
|
||||||
response = r.json()
|
}
|
||||||
logging.info(response['verbose_msg'] + " " + response['permalink'])
|
needs_file_upload = False
|
||||||
|
while True:
|
||||||
|
r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?'
|
||||||
|
+ urllib.parse.urlencode(data), headers=headers)
|
||||||
|
if r.status_code == 200:
|
||||||
|
response = r.json()
|
||||||
|
if response['response_code'] == 0:
|
||||||
|
needs_file_upload = True
|
||||||
|
else:
|
||||||
|
response['filename'] = apkName
|
||||||
|
response['packageName'] = packageName
|
||||||
|
response['versionCode'] = versionCode
|
||||||
|
if kwargs.get('versionName'):
|
||||||
|
response['versionName'] = kwargs.get('versionName')
|
||||||
|
with open(outputfilename, 'w') as fp:
|
||||||
|
json.dump(response, fp, indent=2, sort_keys=True)
|
||||||
|
|
||||||
|
if response.get('positives', 0) > 0:
|
||||||
|
logging.warning(repofilename + ' has been flagged by virustotal '
|
||||||
|
+ str(response['positives']) + ' times:'
|
||||||
|
+ '\n\t' + response['permalink'])
|
||||||
|
break
|
||||||
|
elif r.status_code == 204:
|
||||||
|
time.sleep(10) # wait for public API rate limiting
|
||||||
|
|
||||||
|
upload_url = None
|
||||||
|
if needs_file_upload:
|
||||||
|
manual_url = 'https://www.virustotal.com/'
|
||||||
|
size = os.path.getsize(repofilename)
|
||||||
|
if size > 200000000:
|
||||||
|
# VirusTotal API 200MB hard limit
|
||||||
|
logging.error(_('{path} more than 200MB, manually upload: {url}')
|
||||||
|
.format(path=repofilename, url=manual_url))
|
||||||
|
elif size > 32000000:
|
||||||
|
# VirusTotal API requires fetching a URL to upload bigger files
|
||||||
|
r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?'
|
||||||
|
+ urllib.parse.urlencode(data), headers=headers)
|
||||||
|
if r.status_code == 200:
|
||||||
|
upload_url = r.json().get('upload_url')
|
||||||
|
elif r.status_code == 403:
|
||||||
|
logging.error(_('VirusTotal API key cannot upload files larger than 32MB, '
|
||||||
|
+ 'use {url} to upload {path}.')
|
||||||
|
.format(path=repofilename, url=manual_url))
|
||||||
|
else:
|
||||||
|
r.raise_for_status()
|
||||||
|
else:
|
||||||
|
upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan'
|
||||||
|
|
||||||
|
if upload_url:
|
||||||
|
logging.info(_('Uploading {apkfilename} to virustotal')
|
||||||
|
.format(apkfilename=repofilename))
|
||||||
|
files = {
|
||||||
|
'file': (apkName, open(repofilename, 'rb'))
|
||||||
|
}
|
||||||
|
r = requests.post(upload_url, data=data, headers=headers, files=files)
|
||||||
|
logging.debug(_('If this upload fails, try manually uploading to {url}')
|
||||||
|
.format(url=manual_url))
|
||||||
|
r.raise_for_status()
|
||||||
|
response = r.json()
|
||||||
|
logging.info(response['verbose_msg'] + " " + response['permalink'])
|
||||||
|
|
||||||
|
return outputfilename
|
||||||
|
|
||||||
|
|
||||||
def push_binary_transparency(git_repo_path, git_remote):
|
def push_binary_transparency(git_repo_path, git_remote):
|
||||||
|
@ -142,6 +142,12 @@ class ServerTest(unittest.TestCase):
|
|||||||
repo_section)
|
repo_section)
|
||||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||||
|
|
||||||
|
@unittest.skipIf(not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set')
|
||||||
|
def test_upload_to_virustotal(self):
|
||||||
|
fdroidserver.server.options.verbose = True
|
||||||
|
virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
|
||||||
|
fdroidserver.server.upload_to_virustotal('repo', virustotal_apikey)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
os.chdir(os.path.dirname(__file__))
|
os.chdir(os.path.dirname(__file__))
|
||||||
|
Loading…
Reference in New Issue
Block a user