|
|
@ -78,9 +78,18 @@ class DeviantartExtractor(Extractor):
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
self.user = profile["user"]["username"]
|
|
|
|
self.user = profile["user"]["username"]
|
|
|
|
|
|
|
|
|
|
|
|
if self.extra:
|
|
|
|
extra = self.extra
|
|
|
|
finditer_stash = DeviantartStashExtractor.pattern.finditer
|
|
|
|
if extra:
|
|
|
|
finditer_deviation = DeviantartDeviationExtractor.pattern.finditer
|
|
|
|
if extra == "stash":
|
|
|
|
|
|
|
|
extra_stash = DeviantartStashExtractor.pattern.finditer
|
|
|
|
|
|
|
|
extra_deviation = None
|
|
|
|
|
|
|
|
elif extra == "deviations":
|
|
|
|
|
|
|
|
extra_deviation = DeviantartDeviationExtractor.pattern.finditer
|
|
|
|
|
|
|
|
extra_stash = None
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
extra_stash = DeviantartStashExtractor.pattern.finditer
|
|
|
|
|
|
|
|
extra_deviation = DeviantartDeviationExtractor.pattern.finditer
|
|
|
|
|
|
|
|
extra = True
|
|
|
|
|
|
|
|
|
|
|
|
yield Message.Version, 1
|
|
|
|
yield Message.Version, 1
|
|
|
|
for deviation in self.deviations():
|
|
|
|
for deviation in self.deviations():
|
|
|
@ -131,21 +140,23 @@ class DeviantartExtractor(Extractor):
|
|
|
|
|
|
|
|
|
|
|
|
if "excerpt" in deviation and self.commit_journal:
|
|
|
|
if "excerpt" in deviation and self.commit_journal:
|
|
|
|
journal = self.api.deviation_content(deviation["deviationid"])
|
|
|
|
journal = self.api.deviation_content(deviation["deviationid"])
|
|
|
|
if self.extra:
|
|
|
|
if extra:
|
|
|
|
deviation["_journal"] = journal["html"]
|
|
|
|
deviation["_journal"] = journal["html"]
|
|
|
|
yield self.commit_journal(deviation, journal)
|
|
|
|
yield self.commit_journal(deviation, journal)
|
|
|
|
|
|
|
|
|
|
|
|
if self.extra:
|
|
|
|
if extra:
|
|
|
|
txt = (deviation.get("description", "") +
|
|
|
|
txt = (deviation.get("description", "") +
|
|
|
|
deviation.get("_journal", ""))
|
|
|
|
deviation.get("_journal", ""))
|
|
|
|
for match in finditer_stash(txt):
|
|
|
|
if extra_stash:
|
|
|
|
url = text.ensure_http_scheme(match.group(0))
|
|
|
|
for match in extra_stash(txt):
|
|
|
|
deviation["_extractor"] = DeviantartStashExtractor
|
|
|
|
url = text.ensure_http_scheme(match.group(0))
|
|
|
|
yield Message.Queue, url, deviation
|
|
|
|
deviation["_extractor"] = DeviantartStashExtractor
|
|
|
|
for match in finditer_deviation(txt):
|
|
|
|
yield Message.Queue, url, deviation
|
|
|
|
url = text.ensure_http_scheme(match.group(0))
|
|
|
|
if extra_deviation:
|
|
|
|
deviation["_extractor"] = DeviantartDeviationExtractor
|
|
|
|
for match in extra_deviation(txt):
|
|
|
|
yield Message.Queue, url, deviation
|
|
|
|
url = text.ensure_http_scheme(match.group(0))
|
|
|
|
|
|
|
|
deviation["_extractor"] = DeviantartDeviationExtractor
|
|
|
|
|
|
|
|
yield Message.Queue, url, deviation
|
|
|
|
|
|
|
|
|
|
|
|
def deviations(self):
|
|
|
|
def deviations(self):
|
|
|
|
"""Return an iterable containing all relevant Deviation-objects"""
|
|
|
|
"""Return an iterable containing all relevant Deviation-objects"""
|
|
|
|