From 4b967fa189107cc593218de603a8df76713ebff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 25 Apr 2017 17:12:48 +0200 Subject: [PATCH] implement and use extractor.config() method --- gallery_dl/extractor/batoto.py | 6 +++--- gallery_dl/extractor/common.py | 4 ++++ gallery_dl/extractor/exhentai.py | 19 ++++++++----------- gallery_dl/extractor/gelbooru.py | 3 +-- gallery_dl/extractor/nijie.py | 6 +++--- gallery_dl/extractor/pixiv.py | 16 +++++++--------- gallery_dl/extractor/seiga.py | 6 +++--- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index ae0681bc..4f19ebe1 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -9,7 +9,7 @@ """Extract manga chapters from https://bato.to/""" from .common import Extractor, AsynchronousExtractor, Message -from .. import text, util, config, exception +from .. import text, util, exception from ..cache import cache import re @@ -21,8 +21,8 @@ class BatotoExtractor(Extractor): def login(self): """Login and set necessary cookies""" - username = config.interpolate(("extractor", "batoto", "username")) - password = config.interpolate(("extractor", "batoto", "password")) + username = self.config("username") + password = self.config("password") if username: cookies = self._login_impl(username, password) for key, value in cookies.items(): diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 5d692b74..9525a7f8 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -38,6 +38,10 @@ class Extractor(): def skip(self, num): return 0 + def config(self, key, default=None): + return config.interpolate( + ("extractor", self.category, self.subcategory, key), default) + def request(self, url, encoding=None, *args, **kwargs): response = safe_request(self.session, url, *args, **kwargs) if encoding: diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 44fc82a7..43ef917e 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -9,7 +9,7 @@ """Extract images from galleries at https://exhentai.org/""" from .common import Extractor, Message -from .. import config, text, util, exception +from .. import text, util, exception from ..cache import cache import time import random @@ -42,12 +42,9 @@ class ExhentaiGalleryExtractor(Extractor): self.key = {} self.count = 0 self.version, self.gid, self.token = match.groups() - self.original = config.interpolate( - ("extractor", "exhentai", "original"), True) - self.wait_min = config.interpolate( - ("extractor", "exhentai", "wait-min"), 3) - self.wait_max = config.interpolate( - ("extractor", "exhentai", "wait-max"), 6) + self.original = self.config("original", True) + self.wait_min = self.config("wait-min", 3) + self.wait_max = self.config("wait-max", 6) if self.wait_max < self.wait_min: self.wait_max = self.wait_min @@ -184,8 +181,8 @@ class ExhentaiGalleryExtractor(Extractor): def login(self): """Login and set necessary cookies""" - username = config.interpolate(("extractor", "exhentai", "username")) - password = config.interpolate(("extractor", "exhentai", "password")) + username = self.config("username") + password = self.config("password") cookies = self._login_impl(username, password) for key, value in cookies.items(): self.session.cookies.set( @@ -198,7 +195,7 @@ class ExhentaiGalleryExtractor(Extractor): cnames = ["ipb_member_id", "ipb_pass_hash"] try: - cookies = config.get(("extractor", "exhentai", "cookies")) + cookies = self.config("cookies") if isinstance(cookies, dict) and all(c in cookies for c in cnames): return cookies except TypeError: @@ -213,7 +210,7 @@ class ExhentaiGalleryExtractor(Extractor): "PassWord": password, "ipb_login_submit": "Login!", } - referer = "http://e-hentai.org/bounce_login.php?b=d&bt=1-1" + referer = "https://e-hentai.org/bounce_login.php?b=d&bt=1-1" self.session.headers["Referer"] = referer response = self.session.post(url, data=params) diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 2a21e25e..859d926a 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -9,7 +9,6 @@ """Extract images from https://gelbooru.com/""" from . import booru -from .. import config class GelbooruExtractor(booru.XMLBooruExtractor): @@ -22,7 +21,7 @@ class GelbooruExtractor(booru.XMLBooruExtractor): def setup(self): self.params.update({"page": "dapi", "s": "post", "q": "index"}) try: - cookies = config.get(("extractor", self.category, "cookies")) + cookies = self.config("cookies") self.session.cookies.update({ key: str(value) for key, value in cookies.items() }) diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index 3658762e..c5f4e22e 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -9,7 +9,7 @@ """Extract images from https://nijie.info/""" from .common import AsynchronousExtractor, Message -from .. import config, text, exception +from .. import text, exception from ..cache import cache @@ -62,8 +62,8 @@ class NijieExtractor(AsynchronousExtractor): def login(self): """Login and obtain session cookie""" - username = config.interpolate(("extractor", "nijie", "username")) - password = config.interpolate(("extractor", "nijie", "password")) + username = self.config("username") + password = self.config("password") self.session.cookies = self._login_impl(username, password) @cache(maxage=30*24*60*60, keyarg=1) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 4b636935..38828b16 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -9,10 +9,9 @@ """Extract images and ugoira from https://www.pixiv.net/""" from .common import Extractor, Message -from .. import config, text, exception +from .. import text, exception from ..cache import cache import re -import json class PixivUserExtractor(Extractor): @@ -39,9 +38,7 @@ class PixivUserExtractor(Extractor): self.artist_id = match.group(1) self.api = PixivAPI(self) self.api_call = self.api.user_works - self.load_ugoira = config.interpolate( - ("extractor", "pixiv", "ugoira"), True - ) + self.load_ugoira = self.config("ugoira", True) def items(self): metadata = self.get_job_metadata() @@ -233,6 +230,8 @@ class PixivAPI(): def __init__(self, extractor): self.session = extractor.session self.log = extractor.log + self.username = extractor.config("username") + self.password = extractor.config("password") self.session.headers.update({ "Referer": "http://www.pixiv.net/", "User-Agent": "PixivIOSApp/5.8.0", @@ -291,9 +290,8 @@ class PixivAPI(): def login(self): """Login and gain a Pixiv Public-API access token""" - username = config.interpolate(("extractor", "pixiv", "username")) - password = config.interpolate(("extractor", "pixiv", "password")) - self.user_id, auth_header = self._login_impl(username, password) + self.user_id, auth_header = self._login_impl( + self.username, self.password) self.session.headers["Authorization"] = auth_header @cache(maxage=50*60, keyarg=1) @@ -323,7 +321,7 @@ class PixivAPI(): @staticmethod def _parse(response, empty=[None]): """Parse a Pixiv Public-API response""" - data = json.loads(response.text) + data = response.json() status = data.get("status") response = data.get("response", empty) if status == "failure" or response == empty: diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index 527511cf..fce82223 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -9,7 +9,7 @@ """Extract images from http://seiga.nicovideo.jp""" from .common import Extractor, Message -from .. import text, config, exception +from .. import text, exception from ..cache import cache from xml.etree import ElementTree @@ -47,8 +47,8 @@ class SeigaExtractor(Extractor): def login(self): """Login and set necessary cookies""" - username = config.interpolate(("extractor", self.category, "username")) - password = config.interpolate(("extractor", self.category, "password")) + username = self.config("username") + password = self.config("password") self.session.cookies = self._login_impl(username, password) @cache(maxage=7*24*60*60, keyarg=1)