Merge branch 'download-index-etag' into 'master'

Support ETag when downloading repository index

See merge request !264

fdroidserver/index.py

@@ -35,9 +35,7 @@ from binascii import hexlify, unhexlify
 from datetime import datetime
 from xml.dom.minidom import Document
 
-import requests
+from fdroidserver import metadata, signindex, common, net
-
-from fdroidserver import metadata, signindex, common
 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
 from fdroidserver.metadata import MetaDataException
 
@@ -557,14 +555,16 @@ class VerificationException(Exception):
     pass
 
 
-def download_repo_index(url_str, verify_fingerprint=True):
+def download_repo_index(url_str, etag=None, verify_fingerprint=True):
     """
     Downloads the repository index from the given :param url_str
     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 
     :raises: VerificationException() if the repository could not be verified
 
-    :return: The index in JSON format.
+    :return: A tuple consisting of:
+        - The index in JSON format or None if the index did not change
+        - The new eTag as returned by the HTTP request
     """
     url = urllib.parse.urlsplit(url_str)
 
@@ -576,11 +576,14 @@ def download_repo_index(url_str, verify_fingerprint=True):
         fingerprint = query['fingerprint'][0]
 
     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
-    r = requests.get(url.geturl())
+    download, new_etag = net.http_get(url.geturl(), etag)
+
+    if download is None:
+        return None, new_etag
 
     with tempfile.NamedTemporaryFile() as fp:
         # write and open JAR file
-        fp.write(r.content)
+        fp.write(download)
         jar = zipfile.ZipFile(fp)
 
         # verify that the JAR signature is valid
@@ -601,7 +604,7 @@ def download_repo_index(url_str, verify_fingerprint=True):
         # turn the apps into App objects
         index["apps"] = [metadata.App(app) for app in index["apps"]]
 
-        return index
+        return index, new_etag
 
 
 def verify_jar_signature(file):

fdroidserver/net.py

@@ -34,3 +34,34 @@ def download_file(url, local_filename=None, dldir='tmp'):
                 f.write(chunk)
                 f.flush()
     return local_filename
+
+
+def http_get(url, etag=None):
+    """
+    Downloads the content from the given URL by making a GET request.
+
+    If an ETag is given, it will do a HEAD request first, to see if the content changed.
+
+    :param url: The URL to download from.
+    :param etag: The last ETag to be used for the request (optional).
+    :return: A tuple consisting of:
+        - The raw content that was downloaded or None if it did not change
+        - The new eTag as returned by the HTTP request
+    """
+    headers = {'User-Agent': 'F-Droid'}
+    # TODO disable TLS Session IDs and TLS Session Tickets
+    #      (plain text cookie visible to anyone who can see the network traffic)
+    if etag:
+        r = requests.head(url, headers=headers)
+        r.raise_for_status()
+        if 'ETag' in r.headers and etag == r.headers['ETag']:
+            return None, etag
+
+    r = requests.get(url, headers=headers)
+    r.raise_for_status()
+
+    new_etag = None
+    if 'ETag' in r.headers:
+        new_etag = r.headers['ETag']
+
+    return r.content, new_etag

tests/index.TestCase

@@ -6,6 +6,9 @@ import os
 import sys
 import unittest
 import zipfile
+from unittest.mock import patch
+
+import requests
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -18,6 +21,9 @@ import fdroidserver.index
 import fdroidserver.signindex
 
 
+GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
+
+
 class IndexTest(unittest.TestCase):
 
     def setUp(self):
@@ -55,9 +61,7 @@ class IndexTest(unittest.TestCase):
                                 '818E469465F96B704E27BE2FEE4C63AB' +
                                 '9F83DDF30E7A34C7371A4728D83B0BC1')
             if f == 'guardianproject.jar':
-                self.assertTrue(fingerprint ==
+                self.assertTrue(fingerprint == GP_FINGERPRINT)
-                                'B7C2EEFD8DAC7806AF67DFCD92EB1812' +
-                                '6BC08312A7F2D6F3862E46013C7A6135')
 
     def test_get_public_key_from_jar_fails(self):
         basedir = os.path.dirname(__file__)
@@ -72,10 +76,43 @@ class IndexTest(unittest.TestCase):
             fdroidserver.index.download_repo_index("http://example.org")
 
     def test_download_repo_index_no_jar(self):
-        with self.assertRaises(zipfile.BadZipFile):
+        with self.assertRaises(requests.exceptions.HTTPError):
             fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
 
-    # TODO test_download_repo_index with an actual repository
+    @patch('requests.head')
+    def test_download_repo_index_same_etag(self, head):
+        url = 'http://example.org?fingerprint=test'
+        etag = '"4de5-54d840ce95cb9"'
+
+        head.return_value.headers = {'ETag': etag}
+        index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
+
+        self.assertIsNone(index)
+        self.assertEqual(etag, new_etag)
+
+    @patch('requests.get')
+    @patch('requests.head')
+    def test_download_repo_index_new_etag(self, head, get):
+        url = 'http://example.org?fingerprint=' + GP_FINGERPRINT
+        etag = '"4de5-54d840ce95cb9"'
+
+        # fake HTTP answers
+        head.return_value.headers = {'ETag': 'new_etag'}
+        get.return_value.headers = {'ETag': 'new_etag'}
+        get.return_value.status_code = 200
+        testfile = os.path.join(os.path.dirname(__file__), 'signindex', 'guardianproject-v1.jar')
+        with open(testfile, 'rb') as file:
+            get.return_value.content = file.read()
+
+        index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
+
+        # assert that the index was retrieved properly
+        self.assertEqual('Guardian Project Official Releases', index['repo']['name'])
+        self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint'])
+        self.assertTrue(len(index['repo']['pubkey']) > 500)
+        self.assertEqual(10, len(index['apps']))
+        self.assertEqual(10, len(index['packages']))
+        self.assertEqual('new_etag', new_etag)
 
 
 if __name__ == "__main__":