[common] detect empty HTTP response bodies

gallery_dl/extractor/common.py

@@ -48,7 +48,8 @@ class Extractor():
             ("extractor", self.category, self.subcategory, key), default)
 
     def request(self, url, method="GET", encoding=None, fatal=True, retries=3,
-                *args, **kwargs):
+                allow_empty=False, *args, **kwargs):
+        max_retries = retries
         while True:
             try:
                 response = self.session.request(method, url, *args, **kwargs)
@@ -56,13 +57,15 @@ class Extractor():
                     response.raise_for_status()
                 if encoding:
                     response.encoding = encoding
+                if response.content or allow_empty:
                     return response
+                msg = "empty response body"
             except requests.exceptions.RequestException as exc:
                 msg = exc
-            retries -= 1
             if not retries:
                 raise exception.HttpError(msg)
-            time.sleep(1)
+            time.sleep(1 + max_retries - retries)
+            retries -= 1
 
     def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""

gallery_dl/extractor/hentaifoundry.py

@@ -111,7 +111,7 @@ class HentaifoundryUserExtractor(Extractor):
             "filter_type": 0,
         }
         self.request("https://www.hentai-foundry.com/site/filters",
-                     method="post", data=formdata)
+                     method="post", data=formdata, allow_empty=True)
 
 
 class HentaifoundryImageExtractor(Extractor):

gallery_dl/extractor/nijie.py

@@ -134,7 +134,7 @@ class NijieImageExtractor(NijieExtractor):
 
     def get_job_metadata(self):
         response = self.request(self.popup_url + self.image_id,
-                                allow_redirects=False)
+                                allow_redirects=False, allow_empty=True)
         if 300 <= response.status_code < 400:
             raise exception.NotFoundError("image")
         self.page = response.text

gallery_dl/job.py

@@ -62,7 +62,7 @@ class Job():
             res = str(exc) or "resource (gallery/image/user)"
             log.error("The %s at '%s' does not exist", res, self.url)
         except exception.HttpError as exc:
-            log.error("HTTP request failed:\n%s", exc)
+            log.error("HTTP request failed:  %s", exc)
         except exception.FormatError as exc:
             err, obj = exc.args
             log.error("Applying %s format string failed:  %s: %s",