implement and use extractor.config() method

gallery_dl/extractor/batoto.py

@@ -9,7 +9,7 @@
 """Extract manga chapters from https://bato.to/"""
 
 from .common import Extractor, AsynchronousExtractor, Message
-from .. import text, util, config, exception
+from .. import text, util, exception
 from ..cache import cache
 import re
 
@@ -21,8 +21,8 @@ class BatotoExtractor(Extractor):
 
     def login(self):
         """Login and set necessary cookies"""
-        username = config.interpolate(("extractor", "batoto", "username"))
+        username = self.config("username")
-        password = config.interpolate(("extractor", "batoto", "password"))
+        password = self.config("password")
         if username:
             cookies = self._login_impl(username, password)
             for key, value in cookies.items():

gallery_dl/extractor/common.py

@@ -38,6 +38,10 @@ class Extractor():
     def skip(self, num):
         return 0
 
+    def config(self, key, default=None):
+        return config.interpolate(
+            ("extractor", self.category, self.subcategory, key), default)
+
     def request(self, url, encoding=None, *args, **kwargs):
         response = safe_request(self.session, url, *args, **kwargs)
         if encoding:

gallery_dl/extractor/exhentai.py

@@ -9,7 +9,7 @@
 """Extract images from galleries at https://exhentai.org/"""
 
 from .common import Extractor, Message
-from .. import config, text, util, exception
+from .. import text, util, exception
 from ..cache import cache
 import time
 import random
@@ -42,12 +42,9 @@ class ExhentaiGalleryExtractor(Extractor):
         self.key = {}
         self.count = 0
         self.version, self.gid, self.token = match.groups()
-        self.original = config.interpolate(
+        self.original = self.config("original", True)
-            ("extractor", "exhentai", "original"), True)
+        self.wait_min = self.config("wait-min", 3)
-        self.wait_min = config.interpolate(
+        self.wait_max = self.config("wait-max", 6)
-            ("extractor", "exhentai", "wait-min"), 3)
-        self.wait_max = config.interpolate(
-            ("extractor", "exhentai", "wait-max"), 6)
         if self.wait_max < self.wait_min:
             self.wait_max = self.wait_min
 
@@ -184,8 +181,8 @@ class ExhentaiGalleryExtractor(Extractor):
 
     def login(self):
         """Login and set necessary cookies"""
-        username = config.interpolate(("extractor", "exhentai", "username"))
+        username = self.config("username")
-        password = config.interpolate(("extractor", "exhentai", "password"))
+        password = self.config("password")
         cookies = self._login_impl(username, password)
         for key, value in cookies.items():
             self.session.cookies.set(
@@ -198,7 +195,7 @@ class ExhentaiGalleryExtractor(Extractor):
         cnames = ["ipb_member_id", "ipb_pass_hash"]
 
         try:
-            cookies = config.get(("extractor", "exhentai", "cookies"))
+            cookies = self.config("cookies")
             if isinstance(cookies, dict) and all(c in cookies for c in cnames):
                 return cookies
         except TypeError:
@@ -213,7 +210,7 @@ class ExhentaiGalleryExtractor(Extractor):
             "PassWord": password,
             "ipb_login_submit": "Login!",
         }
-        referer = "http://e-hentai.org/bounce_login.php?b=d&bt=1-1"
+        referer = "https://e-hentai.org/bounce_login.php?b=d&bt=1-1"
         self.session.headers["Referer"] = referer
         response = self.session.post(url, data=params)
 

gallery_dl/extractor/gelbooru.py

@@ -9,7 +9,6 @@
 """Extract images from https://gelbooru.com/"""
 
 from . import booru
-from .. import config
 
 
 class GelbooruExtractor(booru.XMLBooruExtractor):
@@ -22,7 +21,7 @@ class GelbooruExtractor(booru.XMLBooruExtractor):
     def setup(self):
         self.params.update({"page": "dapi", "s": "post", "q": "index"})
         try:
-            cookies = config.get(("extractor", self.category, "cookies"))
+            cookies = self.config("cookies")
             self.session.cookies.update({
                 key: str(value) for key, value in cookies.items()
             })

gallery_dl/extractor/nijie.py

@@ -9,7 +9,7 @@
 """Extract images from https://nijie.info/"""
 
 from .common import AsynchronousExtractor, Message
-from .. import config, text, exception
+from .. import text, exception
 from ..cache import cache
 
 
@@ -62,8 +62,8 @@ class NijieExtractor(AsynchronousExtractor):
 
     def login(self):
         """Login and obtain session cookie"""
-        username = config.interpolate(("extractor", "nijie", "username"))
+        username = self.config("username")
-        password = config.interpolate(("extractor", "nijie", "password"))
+        password = self.config("password")
         self.session.cookies = self._login_impl(username, password)
 
     @cache(maxage=30*24*60*60, keyarg=1)

gallery_dl/extractor/pixiv.py

@@ -9,10 +9,9 @@
 """Extract images and ugoira from https://www.pixiv.net/"""
 
 from .common import Extractor, Message
-from .. import config, text, exception
+from .. import text, exception
 from ..cache import cache
 import re
-import json
 
 
 class PixivUserExtractor(Extractor):
@@ -39,9 +38,7 @@ class PixivUserExtractor(Extractor):
         self.artist_id = match.group(1)
         self.api = PixivAPI(self)
         self.api_call = self.api.user_works
-        self.load_ugoira = config.interpolate(
+        self.load_ugoira = self.config("ugoira", True)
-            ("extractor", "pixiv", "ugoira"), True
-        )
 
     def items(self):
         metadata = self.get_job_metadata()
@@ -233,6 +230,8 @@ class PixivAPI():
     def __init__(self, extractor):
         self.session = extractor.session
         self.log = extractor.log
+        self.username = extractor.config("username")
+        self.password = extractor.config("password")
         self.session.headers.update({
             "Referer": "http://www.pixiv.net/",
             "User-Agent": "PixivIOSApp/5.8.0",
@@ -291,9 +290,8 @@ class PixivAPI():
 
     def login(self):
         """Login and gain a Pixiv Public-API access token"""
-        username = config.interpolate(("extractor", "pixiv", "username"))
+        self.user_id, auth_header = self._login_impl(
-        password = config.interpolate(("extractor", "pixiv", "password"))
+            self.username, self.password)
-        self.user_id, auth_header = self._login_impl(username, password)
         self.session.headers["Authorization"] = auth_header
 
     @cache(maxage=50*60, keyarg=1)
@@ -323,7 +321,7 @@ class PixivAPI():
     @staticmethod
     def _parse(response, empty=[None]):
         """Parse a Pixiv Public-API response"""
-        data = json.loads(response.text)
+        data = response.json()
         status = data.get("status")
         response = data.get("response", empty)
         if status == "failure" or response == empty:

gallery_dl/extractor/seiga.py

@@ -9,7 +9,7 @@
 """Extract images from http://seiga.nicovideo.jp"""
 
 from .common import Extractor, Message
-from .. import text, config, exception
+from .. import text, exception
 from ..cache import cache
 from xml.etree import ElementTree
 
@@ -47,8 +47,8 @@ class SeigaExtractor(Extractor):
 
     def login(self):
         """Login and set necessary cookies"""
-        username = config.interpolate(("extractor", self.category, "username"))
+        username = self.config("username")
-        password = config.interpolate(("extractor", self.category, "password"))
+        password = self.config("password")
         self.session.cookies = self._login_impl(username, password)
 
     @cache(maxage=7*24*60*60, keyarg=1)