|
|
@ -64,6 +64,11 @@ class TwitterExtractor(Extractor):
|
|
|
|
tweets = self._expand_tweets(self.tweets())
|
|
|
|
tweets = self._expand_tweets(self.tweets())
|
|
|
|
self.tweets = lambda : tweets
|
|
|
|
self.tweets = lambda : tweets
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if self.config("unique", True):
|
|
|
|
|
|
|
|
seen_tweets = set()
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
seen_tweets = None
|
|
|
|
|
|
|
|
|
|
|
|
for tweet in self.tweets():
|
|
|
|
for tweet in self.tweets():
|
|
|
|
|
|
|
|
|
|
|
|
if "legacy" in tweet:
|
|
|
|
if "legacy" in tweet:
|
|
|
@ -71,6 +76,11 @@ class TwitterExtractor(Extractor):
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
data = tweet
|
|
|
|
data = tweet
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if seen_tweets is not None:
|
|
|
|
|
|
|
|
if data["id_str"] in seen_tweets:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
seen_tweets.add(data["id_str"])
|
|
|
|
|
|
|
|
|
|
|
|
if not self.retweets and "retweeted_status_id_str" in data:
|
|
|
|
if not self.retweets and "retweeted_status_id_str" in data:
|
|
|
|
self.log.debug("Skipping %s (retweet)", data["id_str"])
|
|
|
|
self.log.debug("Skipping %s (retweet)", data["id_str"])
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|