[deviantart] handle CloudFront blocks in general (#5363)

This was already done for non-OAuth requests (#655)
but CF is now blocking OAuth API requests as well.
pull/5372/head
Mike Fährmann 6 months ago
parent 31e7ca73b6
commit 925123e007
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -84,6 +84,16 @@ class DeviantartExtractor(Extractor):
else:
self.commit_journal = None
def request(self, url, **kwargs):
if "fatal" not in kwargs:
kwargs["fatal"] = False
while True:
response = Extractor.request(self, url, **kwargs)
if response.status_code != 403 or \
b"Request blocked." not in response.content:
return response
self.wait(seconds=300, reason="CloudFront block")
def skip(self, num):
self.offset += num
return num
@ -462,18 +472,12 @@ class DeviantartExtractor(Extractor):
def _limited_request(self, url, **kwargs):
"""Limits HTTP requests to one every 2 seconds"""
kwargs["fatal"] = None
diff = time.time() - DeviantartExtractor._last_request
if diff < 2.0:
self.sleep(2.0 - diff, "request")
while True:
response = self.request(url, **kwargs)
if response.status_code != 403 or \
b"Request blocked." not in response.content:
DeviantartExtractor._last_request = time.time()
return response
self.wait(seconds=180)
response = self.request(url, **kwargs)
DeviantartExtractor._last_request = time.time()
return response
def _fetch_premium(self, deviation):
try:
@ -1418,9 +1422,14 @@ class DeviantartOAuthAPI():
self.authenticate(None if public else self.refresh_token_key)
kwargs["headers"] = self.headers
response = self.extractor.request(url, **kwargs)
data = response.json()
status = response.status_code
try:
data = response.json()
except ValueError:
self.log.error("Unable to parse API response")
data = {}
status = response.status_code
if 200 <= status < 400:
if self.delay > self.delay_min:
self.delay -= 1

Loading…
Cancel
Save