From 8f8cf92ae22c2c30ef5e8b868691837afba25387 Mon Sep 17 00:00:00 2001
From: jesopo
Date: Wed, 11 Sep 2019 15:28:13 +0100
Subject: automatically decode certain http content types

---
 src/utils/http.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'src/utils')

diff --git a/src/utils/http.py b/src/utils/http.py
index 9459dcc0..5b5f36bb 100644
--- a/src/utils/http.py
+++ b/src/utils/http.py
@@ -34,6 +34,7 @@ USER_AGENT = ("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
 
 RESPONSE_MAX = (1024*1024)*100
 SOUP_CONTENT_TYPES = ["text/html", "text/xml", "application/xml"]
+DECODE_CONTENT_TYPES = ["text/plain"]+SOUP_CONTENT_TYPES
 
 class HTTPException(Exception):
     pass
@@ -145,7 +146,11 @@ def request(url: str, method: str="GET", get_params: dict={},
         except _json.decoder.JSONDecodeError as e:
             raise HTTPParsingException(str(e), data)
 
-    return Response(response.status_code, response_content, response_headers)
+    if content_type in DECODE_CONTENT_TYPES:
+        return Response(response.status_code, _decode_data(), response_headers)
+    else:
+        return Response(response.status_code, response_content,
+            response_headers)
 
 def request_many(urls: typing.List[str]) -> typing.Dict[str, Response]:
     responses = {}
-- 
cgit v1.3.1-10-gc9f91