diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | src/utils/http.py | 20 |
3 files changed, 21 insertions, 2 deletions
@@ -2,7 +2,7 @@ Python3 event-driven modular IRC bot! ## Requirements -Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once. +Python3.6+ and [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4), [dnspython](https://pypi.org/project/dnspython/), [feedparser](https://pypi.org/project/feedparser/), [lxml](https://pypi.org/project/lxml/), [netifaces](https://pypi.org/project/netifaces/), [requests](https://pypi.org/project/requests/), [scrypt](https://pypi.python.org/pypi/scrypt), [suds](https://pypi.python.org/pypi/suds-jurko), [tornado](https://pypi.org/project/tornado/) and [tweepy](https://pypi.org/project/tweepy/). Use `pip3 install -r requirements.txt` to install them all at once. ## Setup See [docs/help/setup.md](docs/help/setup.md). diff --git a/requirements.txt b/requirements.txt index 75897f2c..a229508e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ netifaces requests scrypt suds-jurko +tornado tweepy diff --git a/src/utils/http.py b/src/utils/http.py index 2885ebee..d12a978b 100644 --- a/src/utils/http.py +++ b/src/utils/http.py @@ -1,7 +1,7 @@ import ipaddress, re, signal, socket, traceback, typing import urllib.error, urllib.parse import json as _json -import bs4, netifaces, requests +import bs4, netifaces, requests, tornado.gen, tornado.httpclient, tornado.ioloop from src import utils REGEX_URL = re.compile("https?://[A-Z0-9{}]+".format(re.escape("-._~:/%?#[]@!$&'()*+,;=")), re.I) @@ -109,6 +109,24 @@ def request(url: str, method: str="GET", get_params: dict={}, return Response(response.status_code, data, response_headers) +def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]: + responses = {} + + @tornado.gen.coroutine + def _request(): + for url in urls: + client = tornado.httpclient.AsyncHTTPClient() + request = tornado.httpclient.HTTPRequest(url, method="GET", + connect_timeout=2, request_timeout=2) + response = yield client.fetch(request) + + headers = utils.CaseInsensitiveDict(dict(response.headers)) + data = response.body.decode("utf8") + responses[url] = Response(response.code, data, headers) + + tornado.ioloop.IOLoop.current().run_sync(_request) + return responses + def strip_html(s: str) -> str: return bs4.BeautifulSoup(s, "lxml").get_text() |