[patreon] implement 'files' option (#1935)

pull/1971/head
Mike Fährmann 3 years ago
parent 6695ef2e10
commit 8d676151b7
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -1577,6 +1577,19 @@ Description
port than the default.
extractor.patreon.files
-----------------------
Type
``list`` of ``strings``
Default
``["images", "attachments", "postfile", "content"]``
Description
Determines the type and order of files to be downloaded.
Available types are
``postfile``, ``images``, ``attachments``, and ``content``.
extractor.photobucket.subalbums
-------------------------------
Type

@ -32,22 +32,19 @@ class PatreonExtractor(Extractor):
if "session_id" not in self.session.cookies:
self.log.warning("no 'session_id' cookie set")
PatreonExtractor._warning = False
generators = self._build_file_generators(self.config("files"))
for post in self.posts():
if not post.get("current_user_can_view", True):
self.log.warning("Not allowed to view post %s", post["id"])
continue
yield Message.Directory, post
post["num"] = 0
hashes = set()
yield Message.Directory, post
for kind, url, name in itertools.chain(
self._images(post),
self._attachments(post),
self._postfile(post),
self._content(post),
):
for kind, url, name in itertools.chain.from_iterable(
g(post) for g in generators):
fhash = self._filehash(url)
if fhash not in hashes or not fhash:
hashes.add(fhash)
@ -154,7 +151,7 @@ class PatreonExtractor(Extractor):
included[file["type"]][file["id"]]
for file in files["data"]
]
return []
return ()
@memcache(keyarg=1)
def _user(self, url):
@ -211,6 +208,20 @@ class PatreonExtractor(Extractor):
"&json-api-version=1.0"
)
def _build_file_generators(self, filetypes):
if filetypes is None:
return (self._images, self._attachments,
self._postfile, self._content)
genmap = {
"images" : self._images,
"attachments": self._attachments,
"postfile" : self._postfile,
"content" : self._content,
}
if isinstance(filetypes, str):
filetypes = filetypes.split(",")
return [genmap[ft] for ft in filetypes]
class PatreonCreatorExtractor(PatreonExtractor):
"""Extractor for a creator's works"""

Loading…
Cancel
Save