diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/utils/http.py | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/src/utils/http.py b/src/utils/http.py index d12a978b..6ee7fdc3 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -1,7 +1,8 @@ -import ipaddress, re, signal, socket, traceback, typing +import asyncio, ipaddress, re, signal, socket, traceback, typing import urllib.error, urllib.parse import json as _json -import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop +import bs4, netifaces, requests +import tornado.httpclient from src import utils REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I) @@ -112,19 +113,19 @@ def request(url: str, method: str="GET", get_params: dict={}, def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]: responses = {} - @tornado.gen.coroutine - def _request(): + async def _request(): for url in urls: client = tornado.httpclient.AsyncHTTPClient() request = tornado.httpclient.HTTPRequest(url, method="GET", connect_timeout=2, request_timeout=2) - response = yield client.fetch(request) + response = await client.fetch(request) headers = utils.CaseInsensitiveDict(dict(response.headers)) data = response.body.decode("utf8") responses[url] = Response(response.code, data, headers) - tornado.ioloop.IOLoop.current().run_sync(_request) + loop = asyncio.get_event_loop() + loop.run_until_complete(_request()) return responses def strip_html(s: str) -> str: |