From ee0ec0eca1c3b31d22cb2df0106191380bf40cba Mon Sep 17 00:00:00 2001
From: jesopo
Date: Mon, 8 Jul 2019 13:46:27 +0100
Subject: switch request_many() to use asyncio.gather

---
 src/utils/http.py | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

(limited to 'src/utils/http.py')

diff --git a/src/utils/http.py b/src/utils/http.py
index a8e8f05e..9eab10da 100644
--- a/src/utils/http.py
+++ b/src/utils/http.py
@@ -116,24 +116,28 @@ def request(url: str, method: str="GET", get_params: dict={},
 def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
     responses = {}
 
-    async def _request():
-        for url in urls:
-            client = tornado.httpclient.AsyncHTTPClient()
-            request = tornado.httpclient.HTTPRequest(url, method="GET",
-                connect_timeout=2, request_timeout=2)
-
-            response = None
-            try:
-                response = await client.fetch(request)
-            except:
-                return
-
-            headers = utils.CaseInsensitiveDict(dict(response.headers))
-            data = response.body.decode("utf8")
-            responses[url] = Response(response.code, data, headers)
+    async def _request(url):
+        client = tornado.httpclient.AsyncHTTPClient()
+        request = tornado.httpclient.HTTPRequest(url, method="GET",
+            connect_timeout=2, request_timeout=2)
+
+        response = None
+        try:
+            response = await client.fetch(request)
+        except:
+            return
+
+        headers = utils.CaseInsensitiveDict(dict(response.headers))
+        data = response.body.decode("utf8")
+        responses[url] = Response(response.code, data, headers)
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(_request())
+    awaits = []
+    for url in urls:
+        awaits.append(_request(url))
+    task = asyncio.gather(*awaits, return_exceptions=True)
+    loop.run_until_complete(task)
+
     return responses
 
 def strip_html(s: str) -> str:
-- 
cgit v1.3.1-10-gc9f91