[twitter] extend 'ratelimit' option (#5532)

allow waiting for a set amount of seconds
pull/5712/head
Mike Fährmann 4 months ago
parent 2cab87c5b6
commit 162d4269ec
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -3987,6 +3987,7 @@ Description
* ``"abort"``: Raise an error and stop extraction
* ``"wait"``: Wait until rate limit reset
* ``"wait:N"``: Wait for ``N`` seconds
extractor.twitter.relogin
@ -3996,10 +3997,10 @@ Type
Default
``true``
Description
| When receiving a "Could not authenticate you" error while logged in with
`username & passeword <extractor.*.username & .password_>`__,
| refresh the current login session and
try to continue from where it left off.
When receiving a "Could not authenticate you" error while logged in with
`username & passeword <extractor.*.username & .password_>`__,
refresh the current login session and
try to continue from where it left off.
extractor.twitter.locked

@ -1709,11 +1709,16 @@ class TwitterAPI():
variables["cursor"] = cursor
def _handle_ratelimit(self, response):
if self.extractor.config("ratelimit") == "abort":
rl = self.extractor.config("ratelimit")
if rl == "abort":
raise exception.StopExtraction("Rate limit exceeded")
until = response.headers.get("x-rate-limit-reset")
self.extractor.wait(until=until, seconds=None if until else 60)
elif rl and isinstance(rl, str) and rl.startswith("wait:"):
until = None
seconds = text.parse_float(rl.partition(":")[2]) or 60.0
else:
until = response.headers.get("x-rate-limit-reset")
seconds = None if until else 60.0
self.extractor.wait(until=until, seconds=seconds)
def _process_tombstone(self, entry, tombstone):
text = (tombstone.get("richText") or tombstone["text"])["text"]

Loading…
Cancel
Save