[twitter] update 'search' pagination (#544)

Only stop when list of all returned Tweets is empty
instead of when no valid Tweet was found.
pull/3414/head
Mike Fährmann 2 years ago
parent 1cbc234819
commit 90a9c0790f
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -1216,15 +1216,16 @@ class TwitterAPI():
original_retweets = (self.extractor.retweets == "original")
while True:
cursor = tweet = None
data = self._call(endpoint, params)
instr = data["timeline"]["instructions"]
if not instr:
return
tweet_ids = []
tweets = data["globalObjects"]["tweets"]
users = data["globalObjects"]["users"]
tweet_id = cursor = None
tweet_ids = []
# collect tweet IDs and cursor value
for entry in instr[0]["addEntries"]["entries"]:
@ -1243,7 +1244,7 @@ class TwitterAPI():
cursor = entry["content"]["operation"]["cursor"]
if not cursor.get("stopOnEmptyResponse", True):
# keep going even if there are no tweets
tweet = True
tweet_id = True
cursor = cursor["value"]
elif entry_startswith("conversationThread-"):
@ -1292,7 +1293,7 @@ class TwitterAPI():
cursor = (instr[-1]["replaceEntry"]["entry"]
["content"]["operation"]["cursor"]["value"])
if not cursor or not tweet:
if not cursor or (not tweets and not tweet_id):
return
params["cursor"] = cursor

Loading…
Cancel
Save