Merge branch 'asynchronous_reader' into 'master'

Add asynchronous filereader, fix python3 lockups

with the current implementation of AsynchronousFileReader with Python 3 FDroidPopen deadlocks with 100% CPU-utilization

The code is from https://github.com/soxofaan/asynchronousfilereader

I hope the MIT-Licence makes no problems




See merge request !80

fdroidserver/asynchronousfilereader/__init__.py

@@ -0,0 +1,64 @@
+"""
+AsynchronousFileReader
+======================
+
+Simple thread based asynchronous file reader for Python.
+
+see https://github.com/soxofaan/asynchronousfilereader
+
+MIT License
+Copyright (c) 2014 Stefaan Lippens
+"""
+
+__version__ = '0.2.1'
+
+import threading
+try:
+    # Python 2
+    from Queue import Queue
+except ImportError:
+    # Python 3
+    from queue import Queue
+
+
+class AsynchronousFileReader(threading.Thread):
+    """
+    Helper class to implement asynchronous reading of a file
+    in a separate thread. Pushes read lines on a queue to
+    be consumed in another thread.
+    """
+
+    def __init__(self, fd, queue=None, autostart=True):
+        self._fd = fd
+        if queue is None:
+            queue = Queue()
+        self.queue = queue
+
+        threading.Thread.__init__(self)
+
+        if autostart:
+            self.start()
+
+    def run(self):
+        """
+        The body of the tread: read lines and put them on the queue.
+        """
+        while True:
+            line = self._fd.readline()
+            if not line:
+                break
+            self.queue.put(line)
+
+    def eof(self):
+        """
+        Check whether there is no more content to expect.
+        """
+        return not self.is_alive() and self.queue.empty()
+
+    def readlines(self):
+        """
+        Get currently available lines.
+        """
+        while not self.queue.empty():
+            yield self.queue.get()
+

fdroidserver/common.py

@@ -30,7 +30,6 @@ import subprocess
 import time
 import operator
 import Queue
-import threading
 import logging
 import hashlib
 import socket
@@ -40,6 +39,8 @@ from distutils.version import LooseVersion
 from zipfile import ZipFile
 
 import metadata
+from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+
 
 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
 
@@ -1547,31 +1548,6 @@ def isApkDebuggable(apkfile, config):
     return False
 
 
-class AsynchronousFileReader(threading.Thread):
-
-    '''
-    Helper class to implement asynchronous reading of a file
-    in a separate thread. Pushes read lines on a queue to
-    be consumed in another thread.
-    '''
-
-    def __init__(self, fd, queue):
-        assert isinstance(queue, Queue.Queue)
-        assert callable(fd.readline)
-        threading.Thread.__init__(self)
-        self._fd = fd
-        self._queue = queue
-
-    def run(self):
-        '''The body of the tread: read lines and put them on the queue.'''
-        for line in iter(self._fd.readline, ''):
-            self._queue.put(line)
-
-    def eof(self):
-        '''Check whether there is no more content to expect.'''
-        return not self.is_alive() and self._queue.empty()
-
-
 class PopenResult:
     returncode = None
     output = ''
@@ -1612,7 +1588,6 @@ def FDroidPopen(commands, cwd=None, output=True):
 
     stdout_queue = Queue.Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
-    stdout_reader.start()
 
     # Check the queue for output (until there is no more to get)
     while not stdout_reader.eof():

setup.py

@@ -16,7 +16,7 @@ setup(name='fdroidserver',
       author='The F-Droid Project',
       author_email='team@f-droid.org',
       url='https://f-droid.org',
-      packages=['fdroidserver'],
+      packages=['fdroidserver', 'fdroidserver.asynchronousfilereader'],
       scripts=['fdroid', 'fd-commit'],
       data_files=[
           (data_prefix + '/share/doc/fdroidserver/examples',