[deviantart] extend 'extra' option

also download from embedded DeviantArt posts
pull/1352/head
Mike Fährmann 4 years ago
parent 560277394e
commit 5ad2b9c82b
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -729,7 +729,7 @@ Type
Default
``false``
Description
Download extra Sta.sh resources from
Download embedded Deviations and Sta.sh resources from
description texts and journals.
Note: Enabling this option also enables deviantart.metadata_.

@ -78,6 +78,10 @@ class DeviantartExtractor(Extractor):
else:
self.user = profile["user"]["username"]
if self.extra:
finditer_stash = DeviantartStashExtractor.pattern.finditer
finditer_deviation = DeviantartDeviationExtractor.pattern.finditer
yield Message.Version, 1
for deviation in self.deviations():
if isinstance(deviation, tuple):
@ -134,10 +138,14 @@ class DeviantartExtractor(Extractor):
if self.extra:
txt = (deviation.get("description", "") +
deviation.get("_journal", ""))
for match in DeviantartStashExtractor.pattern.finditer(txt):
for match in finditer_stash(txt):
url = text.ensure_http_scheme(match.group(0))
deviation["_extractor"] = DeviantartStashExtractor
yield Message.Queue, url, deviation
for match in finditer_deviation(txt):
url = text.ensure_http_scheme(match.group(0))
deviation["_extractor"] = DeviantartDeviationExtractor
yield Message.Queue, url, deviation
def deviations(self):
"""Return an iterable containing all relevant Deviation-objects"""

Loading…
Cancel
Save