diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 0035c3ed..643210b2 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -35,9 +35,7 @@ from binascii import hexlify, unhexlify from datetime import datetime from xml.dom.minidom import Document -import requests - -from fdroidserver import metadata, signindex, common +from fdroidserver import metadata, signindex, common, net from fdroidserver.common import FDroidPopen, FDroidPopenBytes from fdroidserver.metadata import MetaDataException @@ -557,14 +555,16 @@ class VerificationException(Exception): pass -def download_repo_index(url_str, verify_fingerprint=True): +def download_repo_index(url_str, etag=None, verify_fingerprint=True): """ Downloads the repository index from the given :param url_str and verifies the repository's fingerprint if :param verify_fingerprint is not False. :raises: VerificationException() if the repository could not be verified - :return: The index in JSON format. + :return: A tuple consisting of: + - The index in JSON format or None if the index did not change + - The new eTag as returned by the HTTP request """ url = urllib.parse.urlsplit(url_str) @@ -576,11 +576,14 @@ def download_repo_index(url_str, verify_fingerprint=True): fingerprint = query['fingerprint'][0] url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '') - r = requests.get(url.geturl()) + download, new_etag = net.http_get(url.geturl(), etag) + + if download is None: + return None, new_etag with tempfile.NamedTemporaryFile() as fp: # write and open JAR file - fp.write(r.content) + fp.write(download) jar = zipfile.ZipFile(fp) # verify that the JAR signature is valid @@ -601,7 +604,7 @@ def download_repo_index(url_str, verify_fingerprint=True): # turn the apps into App objects index["apps"] = [metadata.App(app) for app in index["apps"]] - return index + return index, new_etag def verify_jar_signature(file): diff --git a/fdroidserver/net.py b/fdroidserver/net.py index f7932440..7e8821ea 100644 --- a/fdroidserver/net.py +++ b/fdroidserver/net.py @@ -34,3 +34,34 @@ def download_file(url, local_filename=None, dldir='tmp'): f.write(chunk) f.flush() return local_filename + + +def http_get(url, etag=None): + """ + Downloads the content from the given URL by making a GET request. + + If an ETag is given, it will do a HEAD request first, to see if the content changed. + + :param url: The URL to download from. + :param etag: The last ETag to be used for the request (optional). + :return: A tuple consisting of: + - The raw content that was downloaded or None if it did not change + - The new eTag as returned by the HTTP request + """ + headers = {'User-Agent': 'F-Droid'} + # TODO disable TLS Session IDs and TLS Session Tickets + # (plain text cookie visible to anyone who can see the network traffic) + if etag: + r = requests.head(url, headers=headers) + r.raise_for_status() + if 'ETag' in r.headers and etag == r.headers['ETag']: + return None, etag + + r = requests.get(url, headers=headers) + r.raise_for_status() + + new_etag = None + if 'ETag' in r.headers: + new_etag = r.headers['ETag'] + + return r.content, new_etag diff --git a/tests/index.TestCase b/tests/index.TestCase index 780da775..2798e781 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -6,6 +6,9 @@ import os import sys import unittest import zipfile +from unittest.mock import patch + +import requests localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) @@ -18,6 +21,9 @@ import fdroidserver.index import fdroidserver.signindex +GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135' + + class IndexTest(unittest.TestCase): def setUp(self): @@ -55,9 +61,7 @@ class IndexTest(unittest.TestCase): '818E469465F96B704E27BE2FEE4C63AB' + '9F83DDF30E7A34C7371A4728D83B0BC1') if f == 'guardianproject.jar': - self.assertTrue(fingerprint == - 'B7C2EEFD8DAC7806AF67DFCD92EB1812' + - '6BC08312A7F2D6F3862E46013C7A6135') + self.assertTrue(fingerprint == GP_FINGERPRINT) def test_get_public_key_from_jar_fails(self): basedir = os.path.dirname(__file__) @@ -72,10 +76,43 @@ class IndexTest(unittest.TestCase): fdroidserver.index.download_repo_index("http://example.org") def test_download_repo_index_no_jar(self): - with self.assertRaises(zipfile.BadZipFile): + with self.assertRaises(requests.exceptions.HTTPError): fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope") - # TODO test_download_repo_index with an actual repository + @patch('requests.head') + def test_download_repo_index_same_etag(self, head): + url = 'http://example.org?fingerprint=test' + etag = '"4de5-54d840ce95cb9"' + + head.return_value.headers = {'ETag': etag} + index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag) + + self.assertIsNone(index) + self.assertEqual(etag, new_etag) + + @patch('requests.get') + @patch('requests.head') + def test_download_repo_index_new_etag(self, head, get): + url = 'http://example.org?fingerprint=' + GP_FINGERPRINT + etag = '"4de5-54d840ce95cb9"' + + # fake HTTP answers + head.return_value.headers = {'ETag': 'new_etag'} + get.return_value.headers = {'ETag': 'new_etag'} + get.return_value.status_code = 200 + testfile = os.path.join(os.path.dirname(__file__), 'signindex', 'guardianproject-v1.jar') + with open(testfile, 'rb') as file: + get.return_value.content = file.read() + + index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag) + + # assert that the index was retrieved properly + self.assertEqual('Guardian Project Official Releases', index['repo']['name']) + self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint']) + self.assertTrue(len(index['repo']['pubkey']) > 500) + self.assertEqual(10, len(index['apps'])) + self.assertEqual(10, len(index['packages'])) + self.assertEqual('new_etag', new_etag) if __name__ == "__main__": diff --git a/tests/signindex/guardianproject-v1.jar b/tests/signindex/guardianproject-v1.jar new file mode 100644 index 00000000..59edc87c Binary files /dev/null and b/tests/signindex/guardianproject-v1.jar differ