From d1ddd525c1b664102d776f0964631f48d0bb9f55 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sat, 22 Oct 2022 23:15:13 +0200 Subject: [PATCH] net.download_file(): retry on errors --- fdroidserver/net.py | 48 +++++++++++++++++++++++++++++++++------------ tests/net.TestCase | 5 +++-- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/fdroidserver/net.py b/fdroidserver/net.py index 688eda68..cf01395a 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -2,6 +2,7 @@ # # net.py - part of the FDroid server tools # Copyright (C) 2015 Hans-Christoph Steiner +# Copyright (C) 2022 FC Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,28 +17,51 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import os import requests +import time import urllib +from requests.adapters import HTTPAdapter, Retry +from requests.exceptions import ChunkedEncodingError HEADERS = {'User-Agent': 'F-Droid'} -def download_file(url, local_filename=None, dldir='tmp'): +def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_factor=0.1): filename = urllib.parse.urlparse(url).path.split('/')[-1] if local_filename is None: local_filename = os.path.join(dldir, filename) - # the stream=True parameter keeps memory usage low - r = requests.get( - url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300 - ) - r.raise_for_status() - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - f.flush() - return local_filename + # Retry applies to failed DNS lookups, socket connections and connection + # timeouts, never to requests where data has made it to the server; so we + # handle ChunkedEncodingError during transfer ourselves. + for i in range(retries + 1): + if retries: + max_retries = Retry(total=retries - i, backoff_factor=backoff_factor) + adapter = HTTPAdapter(max_retries=max_retries) + session = requests.Session() + session.mount('http://', adapter) + session.mount('https://', adapter) + else: + session = requests + # the stream=True parameter keeps memory usage low + r = session.get( + url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300 + ) + r.raise_for_status() + try: + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + return local_filename + except ChunkedEncodingError as err: + if i == retries: + raise err + logging.warning('Download interrupted, retrying...') + time.sleep(backoff_factor * 2**i) + raise ValueError("retries must be >= 0") def http_get(url, etag=None, timeout=600): diff --git a/tests/net.TestCase b/tests/net.TestCase index e5ea6b24..0addc355 100755 --- a/tests/net.TestCase +++ b/tests/net.TestCase @@ -39,13 +39,14 @@ class NetTest(unittest.TestCase): return MagicMock() requests_get.side_effect = _get - f = net.download_file('https://f-droid.org/repo/index-v1.jar') + f = net.download_file('https://f-droid.org/repo/index-v1.jar', retries=0) self.assertTrue(requests_get.called) self.assertTrue(os.path.exists(f)) self.assertEqual('tmp/index-v1.jar', f) f = net.download_file( - 'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f' + 'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f', + retries=0 ) self.assertTrue(requests_get.called) self.assertTrue(os.path.exists(f))