[8muses] support multi-page listings (#305)

pull/359/head
Mike Fährmann 5 years ago
parent f5961ac968
commit 97ef416218
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -21,7 +21,8 @@ class _8musesAlbumExtractor(Extractor):
filename_fmt = "{page:>03}.{extension}" filename_fmt = "{page:>03}.{extension}"
archive_fmt = "{hash}" archive_fmt = "{hash}"
root = "https://www.8muses.com" root = "https://www.8muses.com"
pattern = r"(?:https?://)?(?:www\.)?8muses\.com(/comics/album/[^?&#]+)" pattern = (r"(?:https?://)?(?:www\.)?8muses\.com"
r"(/comics/album/[^?&#]+)(\?[^#]+)?")
test = ( test = (
("https://www.8muses.com/comics/album/Fakku-Comics/santa/Im-Sorry", { ("https://www.8muses.com/comics/album/Fakku-Comics/santa/Im-Sorry", {
"url": "82449d6a26a29204695cba5d52c3ec60170bc159", "url": "82449d6a26a29204695cba5d52c3ec60170bc159",
@ -52,42 +53,59 @@ class _8musesAlbumExtractor(Extractor):
"private": False, "private": False,
}, },
}), }),
("https://www.8muses.com/comics/album/Fakku-Comics/6?sort=az", {
"count": ">= 70",
"keyword": {"name": r"re:^[S-Zs-z]"},
}),
) )
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
self.path = match.group(1) self.path = match.group(1)
self.params = match.group(2) or ""
def items(self): def items(self):
data = self._unobfuscate(text.extract( url = self.root + self.path + self.params
self.request(self.root + self.path).text,
'<script id="ractive-public" type="text/plain">', '</script>')[0]) while True:
data = self._unobfuscate(text.extract(
self.request(url).text,
'id="ractive-public" type="text/plain">', '</script>')[0])
images = data.get("pictures")
if images:
count = len(images)
album = self._make_album(data["album"])
yield Message.Directory, {"album": album, "count": count}
for num, image in enumerate(images, 1):
url = self.root + "/image/fl/" + image["publicUri"]
img = {
"url" : url,
"page" : num,
"hash" : image["publicUri"],
"count" : count,
"album" : album,
"extension": "jpg",
}
yield Message.Url, url, img
if data.get("pictures"): albums = data.get("albums")
images = data["pictures"] if albums:
album = self._make_album(data["album"]) for album in albums:
yield Message.Directory, {"album": album, "count": len(images)} url = self.root + "/comics/album/" + album["permalink"]
for num, image in enumerate(images, 1): album = {
url = self.root + "/image/fl/" + image["publicUri"] "url" : url,
img = { "name" : album["name"],
"url" : url, "private": album["isPrivate"],
"page" : num, }
"hash" : image["publicUri"], yield Message.Queue, url, album
"count" : len(images),
"album" : album,
"extension": "jpg",
}
yield Message.Url, url, img
if data.get("albums"): if data["page"] >= data["pages"]:
for album in data["albums"]: return
url = self.root + "/comics/album/" + album["permalink"] path, _, num = self.path.rstrip("/").rpartition("/")
album = { path = path if num.isdecimal() else self.path
"url" : url, url = "{}{}/{}{}".format(
"name" : album["name"], self.root, path, data["page"] + 1, self.params)
"private": album["isPrivate"],
}
yield Message.Queue, url, album
def _make_album(self, album): def _make_album(self, album):
return { return {

Loading…
Cancel
Save