[tumblr] small improvements

- don't transform inline GIF URLs
- set 'type' parameter for API calls if there is only
  one post type selected
pull/54/head
Mike Fährmann 7 years ago
parent 9296a26eae
commit b14de6ffc2
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -15,6 +15,8 @@ import re
def _original_image(url):
if url.endswith(".gif") and "_inline_" in url:
return url
return re.sub(
(r"https?://\d+\.media\.tumblr\.com"
r"/([0-9a-f]+)/tumblr_([^/?&#.]+)_\d+\.([0-9a-z]+)"),
@ -45,24 +47,14 @@ class TumblrExtractor(Extractor):
self.user = match.group(1)
self.api = TumblrAPI(self)
self.types = self._setup_posttypes()
self.inline = self.config("inline", False)
self.external = self.config("external", False)
types = self.config("posts", ("photo",))
if types == "all":
self.types = POST_TYPES
elif types:
if isinstance(types, str):
types = types.split(",")
self.types = frozenset(types)
invalid = self.types - POST_TYPES
if invalid:
self.log.warning('invalid post types: "%s"',
'", "'.join(sorted(invalid)))
else:
self.types = frozenset()
self.log.warning("no post types selected")
if len(self.types) == 1:
self.api.params["type"] = next(iter(self.types))
elif not self.types:
self.log.warning("no valid post types selected")
def items(self):
blog = self.api.info(self.user)
@ -111,6 +103,27 @@ class TumblrExtractor(Extractor):
def posts(self):
"""Return an iterable containing all relevant posts"""
def _setup_posttypes(self):
types = self.config("posts", ("photo",))
if types == "all":
return POST_TYPES
elif not types:
return frozenset()
else:
if isinstance(types, str):
types = types.split(",")
types = frozenset(types)
invalid = types - POST_TYPES
if invalid:
types = types & POST_TYPES
self.log.warning('invalid post types: "%s"',
'", "'.join(sorted(invalid)))
return types
@staticmethod
def _prepare(url, post):
post["offset"] += 1
@ -152,11 +165,14 @@ class TumblrPostExtractor(TumblrExtractor):
def __init__(self, match):
TumblrExtractor.__init__(self, match)
self.post_id = match.group(2)
self.types = POST_TYPES
def posts(self):
return self.api.posts(self.user, {"id": self.post_id})
@staticmethod
def _setup_posttypes():
return POST_TYPES
class TumblrTagExtractor(TumblrExtractor):
"""Extractor for images from a tumblr-user by tag"""

@ -83,6 +83,7 @@ skip = [
"archivedmoe", "archiveofsins", "thebarchive",
# temporary issues
"mangazuki",
"yeet",
]
# enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1:

Loading…
Cancel
Save