From f18a3a77ef894d6563543fe449321e52b16e93ad Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Thu, 28 Apr 2016 13:59:41 +0200 Subject: [PATCH] Use new settings to fine tune the connection pool In some circumstances, it is necessary to increase size of pool. --- searx/poolrequests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 13c6a906e..54e8dbcf6 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) +connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): - http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) - https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: - http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) - https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) class SessionSinglePool(requests.Session):