mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
net.download_file(): retry on errors
This commit is contained in:
parent
c61c9f3895
commit
d1ddd525c1
@ -2,6 +2,7 @@
|
|||||||
#
|
#
|
||||||
# net.py - part of the FDroid server tools
|
# net.py - part of the FDroid server tools
|
||||||
# Copyright (C) 2015 Hans-Christoph Steiner <hans@eds.org>
|
# Copyright (C) 2015 Hans-Christoph Steiner <hans@eds.org>
|
||||||
|
# Copyright (C) 2022 FC Stegerman <flx@obfusk.net>
|
||||||
#
|
#
|
||||||
# This program is free software: you can redistribute it and/or modify
|
# This program is free software: you can redistribute it and/or modify
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
@ -16,28 +17,51 @@
|
|||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
|
import time
|
||||||
import urllib
|
import urllib
|
||||||
|
from requests.adapters import HTTPAdapter, Retry
|
||||||
|
from requests.exceptions import ChunkedEncodingError
|
||||||
|
|
||||||
HEADERS = {'User-Agent': 'F-Droid'}
|
HEADERS = {'User-Agent': 'F-Droid'}
|
||||||
|
|
||||||
|
|
||||||
def download_file(url, local_filename=None, dldir='tmp'):
|
def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_factor=0.1):
|
||||||
filename = urllib.parse.urlparse(url).path.split('/')[-1]
|
filename = urllib.parse.urlparse(url).path.split('/')[-1]
|
||||||
if local_filename is None:
|
if local_filename is None:
|
||||||
local_filename = os.path.join(dldir, filename)
|
local_filename = os.path.join(dldir, filename)
|
||||||
|
# Retry applies to failed DNS lookups, socket connections and connection
|
||||||
|
# timeouts, never to requests where data has made it to the server; so we
|
||||||
|
# handle ChunkedEncodingError during transfer ourselves.
|
||||||
|
for i in range(retries + 1):
|
||||||
|
if retries:
|
||||||
|
max_retries = Retry(total=retries - i, backoff_factor=backoff_factor)
|
||||||
|
adapter = HTTPAdapter(max_retries=max_retries)
|
||||||
|
session = requests.Session()
|
||||||
|
session.mount('http://', adapter)
|
||||||
|
session.mount('https://', adapter)
|
||||||
|
else:
|
||||||
|
session = requests
|
||||||
# the stream=True parameter keeps memory usage low
|
# the stream=True parameter keeps memory usage low
|
||||||
r = requests.get(
|
r = session.get(
|
||||||
url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300
|
url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300
|
||||||
)
|
)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
try:
|
||||||
with open(local_filename, 'wb') as f:
|
with open(local_filename, 'wb') as f:
|
||||||
for chunk in r.iter_content(chunk_size=1024):
|
for chunk in r.iter_content(chunk_size=1024):
|
||||||
if chunk: # filter out keep-alive new chunks
|
if chunk: # filter out keep-alive new chunks
|
||||||
f.write(chunk)
|
f.write(chunk)
|
||||||
f.flush()
|
f.flush()
|
||||||
return local_filename
|
return local_filename
|
||||||
|
except ChunkedEncodingError as err:
|
||||||
|
if i == retries:
|
||||||
|
raise err
|
||||||
|
logging.warning('Download interrupted, retrying...')
|
||||||
|
time.sleep(backoff_factor * 2**i)
|
||||||
|
raise ValueError("retries must be >= 0")
|
||||||
|
|
||||||
|
|
||||||
def http_get(url, etag=None, timeout=600):
|
def http_get(url, etag=None, timeout=600):
|
||||||
|
@ -39,13 +39,14 @@ class NetTest(unittest.TestCase):
|
|||||||
return MagicMock()
|
return MagicMock()
|
||||||
|
|
||||||
requests_get.side_effect = _get
|
requests_get.side_effect = _get
|
||||||
f = net.download_file('https://f-droid.org/repo/index-v1.jar')
|
f = net.download_file('https://f-droid.org/repo/index-v1.jar', retries=0)
|
||||||
self.assertTrue(requests_get.called)
|
self.assertTrue(requests_get.called)
|
||||||
self.assertTrue(os.path.exists(f))
|
self.assertTrue(os.path.exists(f))
|
||||||
self.assertEqual('tmp/index-v1.jar', f)
|
self.assertEqual('tmp/index-v1.jar', f)
|
||||||
|
|
||||||
f = net.download_file(
|
f = net.download_file(
|
||||||
'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f'
|
'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f',
|
||||||
|
retries=0
|
||||||
)
|
)
|
||||||
self.assertTrue(requests_get.called)
|
self.assertTrue(requests_get.called)
|
||||||
self.assertTrue(os.path.exists(f))
|
self.assertTrue(os.path.exists(f))
|
||||||
|
Loading…
Reference in New Issue
Block a user