|
|
@ -12,6 +12,7 @@ from .common import Extractor, Message
|
|
|
|
from .. import text, exception
|
|
|
|
from .. import text, exception
|
|
|
|
from ..cache import cache, memcache
|
|
|
|
from ..cache import cache, memcache
|
|
|
|
import itertools
|
|
|
|
import itertools
|
|
|
|
|
|
|
|
import mimetypes
|
|
|
|
import datetime
|
|
|
|
import datetime
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
import math
|
|
|
|
import math
|
|
|
@ -70,8 +71,7 @@ class DeviantartExtractor(Extractor):
|
|
|
|
content = deviation["content"]
|
|
|
|
content = deviation["content"]
|
|
|
|
if (self.original and deviation["is_downloadable"] and
|
|
|
|
if (self.original and deviation["is_downloadable"] and
|
|
|
|
content["filesize"] != deviation["download_filesize"]):
|
|
|
|
content["filesize"] != deviation["download_filesize"]):
|
|
|
|
content.update(
|
|
|
|
self._update_content(deviation, content)
|
|
|
|
self.api.deviation_download(deviation["deviationid"]))
|
|
|
|
|
|
|
|
yield self.commit(deviation, content)
|
|
|
|
yield self.commit(deviation, content)
|
|
|
|
|
|
|
|
|
|
|
|
if "videos" in deviation:
|
|
|
|
if "videos" in deviation:
|
|
|
@ -194,6 +194,15 @@ class DeviantartExtractor(Extractor):
|
|
|
|
url = "{}/{}/{}/0/".format(self.root, self.user, category)
|
|
|
|
url = "{}/{}/{}/0/".format(self.root, self.user, category)
|
|
|
|
return [(url + folder["name"], folder) for folder in folders]
|
|
|
|
return [(url + folder["name"], folder) for folder in folders]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _update_content(self, deviation, content):
|
|
|
|
|
|
|
|
data = self.api.deviation_download(deviation["deviationid"])
|
|
|
|
|
|
|
|
if self.original == "images":
|
|
|
|
|
|
|
|
url = data["src"].partition("?")[0]
|
|
|
|
|
|
|
|
mtype = mimetypes.guess_type(url, False)[0]
|
|
|
|
|
|
|
|
if not mtype or not mtype.startswith("image/"):
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
content.update(data)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DeviantartGalleryExtractor(DeviantartExtractor):
|
|
|
|
class DeviantartGalleryExtractor(DeviantartExtractor):
|
|
|
|
"""Extractor for all deviations from an artist's gallery"""
|
|
|
|
"""Extractor for all deviations from an artist's gallery"""
|
|
|
|