use 'extractor.request()' for more HTTP requests

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## Unreleased
+
 ## 1.4.1 - 2018-06-22
 - Added an `ugoira` post-processor to convert  `pixiv` animations to WebM
 - Added `--zip` and `--ugoira-conv` command-line options

gallery_dl/extractor/exhentai.py

@@ -13,7 +13,6 @@ from .. import text, util, exception
 from ..cache import cache
 import time
 import random
-import requests
 
 
 class ExhentaiExtractor(Extractor):
@@ -214,13 +213,8 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
             "showkey": self.key["show"],
         }
         for request["page"] in range(2, self.count + 1):
-            while True:
-                try:
-                    self.wait()
-                    page = self.session.post(api_url, json=request).json()
-                    break
-                except requests.exceptions.ConnectionError:
-                    pass
+            self.wait()
+            page = self.request(api_url, method="POST", json=request).json()
             imgkey = nextkey
             nextkey, pos = text.extract(page["i3"], "'", "'")
             imgurl , pos = text.extract(page["i3"], 'id="img" src="', '"', pos)

gallery_dl/extractor/pinterest.py

@@ -110,7 +110,7 @@ class PinterestPinitExtractor(PinterestExtractor):
         self.url = "https://" + match.group(1)
 
     def items(self):
-        response = self.session.head(self.url)
+        response = self.request(self.url, method="HEAD", allow_redirects=False)
         location = response.headers.get("Location")
         if not location or location in ("https://api.pinterest.com/None",
                                         "https://www.pinterest.com"):

gallery_dl/extractor/pixiv.py

@@ -159,7 +159,9 @@ class PixivMeExtractor(PixivExtractor):
         self.account = match.group(1)
 
     def items(self):
-        response = self.session.head("https://pixiv.me/" + self.account)
+        url = "https://pixiv.me/" + self.account
+        response = self.request(
+            url, method="HEAD", allow_redirects=False, expect=(404,))
         if response.status_code == 404:
             raise exception.NotFoundError("user")
         yield Message.Version, 1

gallery_dl/extractor/seiga.py

@@ -41,7 +41,8 @@ class SeigaExtractor(Extractor):
     def get_image_url(self, image_id):
         """Get url for an image with id 'image_id'"""
         url = "http://seiga.nicovideo.jp/image/source/{}".format(image_id)
-        response = self.session.head(url)
+        response = self.request(
+            url, method="HEAD", allow_redirects=False, expect=(404,))
         if response.status_code == 404:
             raise exception.NotFoundError("image")
         return response.headers["Location"].replace("/o/", "/priv/", 1)

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.4.1"
+__version__ = "1.4.2-dev"

test/test_results.py

@@ -22,8 +22,6 @@ TRAVIS_SKIP = {
 
 # temporary issues, etc.
 BROKEN = {
-    "desuarchive",
-    "rbt",
     "whatisthisimnotgoodwithcomputers",
 }