diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index d2a69100..d7a41bcf 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -43,6 +43,7 @@ class Extractor(): browser = None request_interval = 0.0 request_interval_min = 0.0 + request_interval_429 = 60.0 request_timestamp = 0.0 def __init__(self, match): @@ -203,7 +204,9 @@ class Extractor(): self.log.warning("Cloudflare CAPTCHA") break - if code == 429 and self._interval_429: + if code == 429 and self._handle_429(response): + continue + elif code == 429 and self._interval_429: pass elif code not in retry_codes and code < 500: break @@ -231,6 +234,8 @@ class Extractor(): raise exception.HttpError(msg, response) + _handle_429 = util.false + def wait(self, seconds=None, until=None, adjust=1.0, reason="rate limit"): now = time.time() @@ -324,7 +329,7 @@ class Extractor(): self.request_interval_min, ) self._interval_429 = util.build_duration_func( - self.config("sleep-429", 60), + self.config("sleep-429", self.request_interval_429), ) if self._retries < 0: diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index 38a2d166..6ec44bad 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -7,7 +7,7 @@ """Extractors for https://skeb.jp/""" from .common import Extractor, Message -from .. import text, exception +from .. import text import itertools @@ -31,14 +31,15 @@ class SkebExtractor(Extractor): if "Authorization" not in self.session.headers: self.headers["Authorization"] = "Bearer null" - def request(self, url, **kwargs): - while True: - try: - return Extractor.request(self, url, **kwargs) - except exception.HttpError as exc: - if exc.status == 429 and "request_key" in exc.response.cookies: - continue - raise + def _handle_429(self, response): + if "request_key" in response.cookies: + return True + + request_key = text.extr( + response.text, "request_key=", ";") + if request_key: + self.cookies.set("request_key", request_key, domain="skeb.jp") + return True def items(self): metadata = self.metadata()