[skeb] fix '429 Too Many Requests' errors (#5766)

Introduce '_handle_429' method to make it easier for Extractors to react
to 429 errors regardless of 'sleep-429' settings.
pull/5774/head
Mike Fährmann 3 months ago
parent 60b4541199
commit 11421cf940
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -43,6 +43,7 @@ class Extractor():
browser = None browser = None
request_interval = 0.0 request_interval = 0.0
request_interval_min = 0.0 request_interval_min = 0.0
request_interval_429 = 60.0
request_timestamp = 0.0 request_timestamp = 0.0
def __init__(self, match): def __init__(self, match):
@ -203,7 +204,9 @@ class Extractor():
self.log.warning("Cloudflare CAPTCHA") self.log.warning("Cloudflare CAPTCHA")
break break
if code == 429 and self._interval_429: if code == 429 and self._handle_429(response):
continue
elif code == 429 and self._interval_429:
pass pass
elif code not in retry_codes and code < 500: elif code not in retry_codes and code < 500:
break break
@ -231,6 +234,8 @@ class Extractor():
raise exception.HttpError(msg, response) raise exception.HttpError(msg, response)
_handle_429 = util.false
def wait(self, seconds=None, until=None, adjust=1.0, def wait(self, seconds=None, until=None, adjust=1.0,
reason="rate limit"): reason="rate limit"):
now = time.time() now = time.time()
@ -324,7 +329,7 @@ class Extractor():
self.request_interval_min, self.request_interval_min,
) )
self._interval_429 = util.build_duration_func( self._interval_429 = util.build_duration_func(
self.config("sleep-429", 60), self.config("sleep-429", self.request_interval_429),
) )
if self._retries < 0: if self._retries < 0:

@ -7,7 +7,7 @@
"""Extractors for https://skeb.jp/""" """Extractors for https://skeb.jp/"""
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text
import itertools import itertools
@ -31,14 +31,15 @@ class SkebExtractor(Extractor):
if "Authorization" not in self.session.headers: if "Authorization" not in self.session.headers:
self.headers["Authorization"] = "Bearer null" self.headers["Authorization"] = "Bearer null"
def request(self, url, **kwargs): def _handle_429(self, response):
while True: if "request_key" in response.cookies:
try: return True
return Extractor.request(self, url, **kwargs)
except exception.HttpError as exc: request_key = text.extr(
if exc.status == 429 and "request_key" in exc.response.cookies: response.text, "request_key=", ";")
continue if request_key:
raise self.cookies.set("request_key", request_key, domain="skeb.jp")
return True
def items(self): def items(self):
metadata = self.metadata() metadata = self.metadata()

Loading…
Cancel
Save