From 9c138dfc1f8a02ab0a1eaa5e2a968fd9613be534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Mon, 25 Sep 2017 13:01:10 +0200
Subject: [PATCH] [common] detect empty HTTP response bodies

---
 gallery_dl/extractor/common.py        | 11 +++++++----
 gallery_dl/extractor/hentaifoundry.py |  2 +-
 gallery_dl/extractor/nijie.py         |  2 +-
 gallery_dl/job.py                     |  2 +-
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 34a3e708..789e81b1 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -48,7 +48,8 @@ class Extractor():
             ("extractor", self.category, self.subcategory, key), default)
 
     def request(self, url, method="GET", encoding=None, fatal=True, retries=3,
-                *args, **kwargs):
+                allow_empty=False, *args, **kwargs):
+        max_retries = retries
         while True:
             try:
                 response = self.session.request(method, url, *args, **kwargs)
@@ -56,13 +57,15 @@ class Extractor():
                     response.raise_for_status()
                 if encoding:
                     response.encoding = encoding
-                return response
+                if response.content or allow_empty:
+                    return response
+                msg = "empty response body"
             except requests.exceptions.RequestException as exc:
                 msg = exc
-            retries -= 1
             if not retries:
                 raise exception.HttpError(msg)
-            time.sleep(1)
+            time.sleep(1 + max_retries - retries)
+            retries -= 1
 
     def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""
diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py
index 2cc3fa7d..2d4cc8bb 100644
--- a/gallery_dl/extractor/hentaifoundry.py
+++ b/gallery_dl/extractor/hentaifoundry.py
@@ -111,7 +111,7 @@ class HentaifoundryUserExtractor(Extractor):
             "filter_type": 0,
         }
         self.request("https://www.hentai-foundry.com/site/filters",
-                     method="post", data=formdata)
+                     method="post", data=formdata, allow_empty=True)
 
 
 class HentaifoundryImageExtractor(Extractor):
diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py
index 1ca5e457..6ee6931a 100644
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@@ -134,7 +134,7 @@ class NijieImageExtractor(NijieExtractor):
 
     def get_job_metadata(self):
         response = self.request(self.popup_url + self.image_id,
-                                allow_redirects=False)
+                                allow_redirects=False, allow_empty=True)
         if 300 <= response.status_code < 400:
             raise exception.NotFoundError("image")
         self.page = response.text
diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 4e321015..795a5c84 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -62,7 +62,7 @@ class Job():
             res = str(exc) or "resource (gallery/image/user)"
             log.error("The %s at '%s' does not exist", res, self.url)
         except exception.HttpError as exc:
-            log.error("HTTP request failed:\n%s", exc)
+            log.error("HTTP request failed:  %s", exc)
         except exception.FormatError as exc:
             err, obj = exc.args
             log.error("Applying %s format string failed:  %s: %s",