From a2aef721d81eeb76b5272b544f96e81d164ad7ee Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 18 Dec 2018 21:48:55 +0100 Subject: [PATCH 1/5] add timeout to net.http_get() and index.download_repo_index() The requests docs recommend this: http://docs.python-requests.org/en/master/user/quickstart/#timeouts And mirror-monitor was hanging forever on a bad mirror. --- fdroidserver/index.py | 4 ++-- fdroidserver/net.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 1ce940b3..7585563f 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -686,7 +686,7 @@ def get_mirror_service_urls(url): return urls -def download_repo_index(url_str, etag=None, verify_fingerprint=True): +def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600): """Downloads and verifies index file, then returns its data. Downloads the repository index from the given :param url_str and @@ -710,7 +710,7 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): fingerprint = query['fingerprint'][0] url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '') - download, new_etag = net.http_get(url.geturl(), etag) + download, new_etag = net.http_get(url.geturl(), etag, timeout) if download is None: return None, new_etag diff --git a/fdroidserver/net.py b/fdroidserver/net.py index 7e8821ea..b9ddf72b 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -36,7 +36,7 @@ def download_file(url, local_filename=None, dldir='tmp'): return local_filename -def http_get(url, etag=None): +def http_get(url, etag=None, timeout=600): """ Downloads the content from the given URL by making a GET request. @@ -52,12 +52,12 @@ def http_get(url, etag=None): # TODO disable TLS Session IDs and TLS Session Tickets # (plain text cookie visible to anyone who can see the network traffic) if etag: - r = requests.head(url, headers=headers) + r = requests.head(url, headers=headers, timeout=timeout) r.raise_for_status() if 'ETag' in r.headers and etag == r.headers['ETag']: return None, etag - r = requests.get(url, headers=headers) + r = requests.get(url, headers=headers, timeout=timeout) r.raise_for_status() new_etag = None From 4c5864c97533e47b5ed7e5a7764fd628c63f3134 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 13 Dec 2018 22:02:46 +0100 Subject: [PATCH 2/5] buildserver: add config option to use 'nfs' instead of '9p' '9p' is not possible when running KVM in VMware. Also, 'nfs' might provide more reliably operation on systems that are properly setup for it. --- buildserver/Vagrantfile | 8 ++++++-- examples/makebuildserver.config.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 013b5382..392d3df6 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -43,8 +43,12 @@ Vagrant.configure("2") do |config| libvirt.nic_model_type = configfile["libvirt_nic_model_type"] end end - config.vm.synced_folder './', '/vagrant', type: '9p' - synced_folder_type = '9p' + if configfile.has_key? "synced_folder_type" + synced_folder_type = configfile["synced_folder_type"] + else + synced_folder_type = '9p' + end + config.vm.synced_folder './', '/vagrant', type: synced_folder_type else abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!") end diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 9d1dae31..cb47f95f 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -85,3 +85,10 @@ # # libvirt_disk_bus = 'sata' # libvirt_nic_model_type = 'rtl8139' + +# Sometimes, it is not possible to use the 9p synced folder type with +# libvirt, like if running a KVM buildserver instance inside of a +# VMware ESXi guest. In that case, using NFS or another method is +# required. +# +# synced_folder_type = 'nfs' From 78473e7dab07b4bb923d7c388941307503b7ce12 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 3 Jan 2019 20:24:35 +0100 Subject: [PATCH 3/5] lint: check if CurrentVersion is older than oldest build entry This causes F-Droid to never install this app, except for manually via the Versions list in AppDetails. --- fdroidserver/lint.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 9eaf4b19..63cf1293 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -525,6 +525,35 @@ def check_for_unsupported_metadata_files(basedir=""): return return_value +def check_current_version_code(app): + """Check that the CurrentVersionCode is currently available""" + + archive_policy = app.get('ArchivePolicy') + if archive_policy and archive_policy.split()[0] == "0": + return + cv = app.get('CurrentVersionCode') + if cv is not None and int(cv) == 0: + return + + builds = app.get('builds') + active_builds = 0 + min_versionCode = None + if builds: + for build in builds: + vc = int(build['versionCode']) + if min_versionCode is None or min_versionCode > vc: + min_versionCode = vc + if not build.get('disable'): + active_builds += 1 + if cv == build['versionCode']: + break + if active_builds == 0: + return # all builds are disabled + if cv is not None and int(cv) < min_versionCode: + yield(_('CurrentVersionCode {cv} is less than oldest build entry {versionCode}') + .format(cv=cv, versionCode=min_versionCode)) + + def main(): global config, options @@ -579,6 +608,7 @@ def main(): check_files_dir, check_format, check_license_tag, + check_current_version_code, ] for check_func in app_check_funcs: From 76b026e572bc0ec2467579d56447addc392e4deb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 3 Jan 2019 20:48:08 +0100 Subject: [PATCH 4/5] lint: check whether the locale was included in an f-droid.org URL --- fdroidserver/lint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 63cf1293..465954cc 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -165,6 +165,8 @@ regex_checks = { 'Description': https_enforcings + http_url_shorteners + [ (re.compile(r'\s*[*#][^ .]'), _("Invalid bulleted list")), + (re.compile(r'https://f-droid.org/[a-z][a-z](_[A-Za-z]{2,4})?/'), + _("Locale included in f-droid.org URL")), (re.compile(r'^\s'), _("Unnecessary leading space")), (re.compile(r'.*\s$'), From 13bcc476a7286457b28605421dc934c2569e3260 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 3 Jan 2019 20:50:57 +0100 Subject: [PATCH 5/5] androguard>=3.3 fails: "AXMLParser' object has no attribute 'is_valid'" Traceback (most recent call last): File "/builds/eighthave/fdroidserver/tests/build.TestCase", line 120, in test_get_apk_metadata vc, vn = fdroidserver.build.get_metadata_from_apk(app, build, apkfilename) File "/builds/eighthave/fdroidserver/fdroidserver/build.py", line 332, in get_metadata_from_apk appid, versionCode, versionName = common.get_apk_id(apkfile) File "/builds/eighthave/fdroidserver/fdroidserver/common.py", line 2126, in get_apk_id return get_apk_id_androguard(apkfile) File "/builds/eighthave/fdroidserver/fdroidserver/common.py", line 2158, in get_apk_id_androguard while axml.is_valid(): AttributeError: 'AXMLParser' object has no attribute 'is_valid' --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a40a4d9..30f2d37b 100755 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup(name='fdroidserver', 'babel', ], install_requires=[ - 'androguard >= 3.1.0rc2', + 'androguard >= 3.1.0rc2, < 3.3.0', 'clint', 'defusedxml', 'GitPython',