Cache apk info to speed up updates

fdroidserver/update.py

@@ -25,6 +25,7 @@ import subprocess
 import re
 import zipfile
 import hashlib
+import pickle
 from xml.dom.minidom import Document
 from optparse import OptionParser
 import time
@@ -217,7 +218,15 @@ def main():
     # Read known apks data (will be updated and written back when we've finished)
     knownapks = common.KnownApks()
 
-    # Gather information about all the apk files in the repo directory...
+    # Gather information about all the apk files in the repo directory, using
+    # cached data if possible.
+    apkcachefile = os.path.join('tmp', 'apkcache')
+    if os.path.exists(apkcachefile):
+        with open(apkcachefile, 'rb') as cf:
+            apkcache = pickle.load(cf)
+    else:
+        apkcache = {}
+    cachechanged = False
     apks = []
     for apkfile in glob.glob(os.path.join('repo','*.apk')):
 
@@ -227,6 +236,13 @@ def main():
             sys.exit(1)
         srcfilename = apkfilename[:-4] + "_src.tar.gz"
 
+        if apkcache.has_key(apkfilename):
+            if options.verbose:
+                print "Reading " + apkfilename + " from cache"
+            thisinfo = apkcache[apkfilename]
+
+        else:
+
             if not options.quiet:
                 print "Processing " + apkfilename
             thisinfo = {}
@@ -288,7 +304,7 @@ def main():
             # Calculate the md5 and sha256...
             m = hashlib.md5()
             sha = hashlib.sha256()
-        f = open(apkfile, 'rb')
+            with open(apkfile, 'rb') as f:
                 while True:
                     t = f.read(1024)
                     if len(t) == 0:
@@ -297,7 +313,6 @@ def main():
                     sha.update(t)
                 thisinfo['md5'] = m.hexdigest()
                 thisinfo['sha256'] = sha.hexdigest()
-        f.close()
 
             # Get the signature (or md5 of, to be precise)...
             p = subprocess.Popen(['java', 'getsig',
@@ -331,8 +346,15 @@ def main():
             if added:
                 thisinfo['added'] = added
 
+            apkcache[apkfilename] = thisinfo
+            cachechanged = True
+
         apks.append(thisinfo)
 
+    if cachechanged:
+        with open(apkcachefile, 'wb') as cf:
+            pickle.dump(apkcache, cf)
+
     # Some information from the apks needs to be applied up to the application
     # level. When doing this, we use the info from the most recent version's apk.
     # We deal with figuring out when the app was added and last updated at the