split e621 from danbooru module (#3425)
Mike Fährmann
2 years ago
parent
1ae48a54f8
commit
925b467496
@ -0,0 +1,196 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014-2023 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
"""Extractors for https://e621.net/ and other e621 instances"""
|
||||
|
||||
from .common import Extractor, Message
|
||||
from . import danbooru
|
||||
from .. import text, version
|
||||
|
||||
|
||||
class E621Extractor(danbooru.DanbooruExtractor):
|
||||
"""Base class for e621 extractors"""
|
||||
basecategory = "E621"
|
||||
page_limit = 750
|
||||
page_start = None
|
||||
per_page = 320
|
||||
request_interval_min = 1.0
|
||||
|
||||
def request(self, url, **kwargs):
|
||||
kwargs["headers"] = self.headers
|
||||
return Extractor.request(self, url, **kwargs)
|
||||
|
||||
def items(self):
|
||||
self.headers = {"User-Agent": "gallery-dl/{} (by mikf)".format(
|
||||
version.__version__)}
|
||||
|
||||
data = self.metadata()
|
||||
for post in self.posts():
|
||||
file = post["file"]
|
||||
|
||||
if not file["url"]:
|
||||
md5 = file["md5"]
|
||||
file["url"] = "https://static1.{}/data/{}/{}/{}.{}".format(
|
||||
self.root[8:], md5[0:2], md5[2:4], md5, file["ext"])
|
||||
|
||||
post["filename"] = file["md5"]
|
||||
post["extension"] = file["ext"]
|
||||
|
||||
post.update(data)
|
||||
yield Message.Directory, post
|
||||
yield Message.Url, file["url"], post
|
||||
|
||||
|
||||
BASE_PATTERN = E621Extractor.update({
|
||||
"e621": {
|
||||
"root": "https://e621.net",
|
||||
"pattern": r"e621\.net",
|
||||
},
|
||||
"e926": {
|
||||
"root": "https://e926.net",
|
||||
"pattern": r"e926\.net",
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor):
|
||||
"""Extractor for e621 posts from tag searches"""
|
||||
pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)"
|
||||
test = (
|
||||
("https://e621.net/posts?tags=anry", {
|
||||
"url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba",
|
||||
"content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58",
|
||||
}),
|
||||
("https://e621.net/post/index/1/anry"),
|
||||
("https://e621.net/post?tags=anry"),
|
||||
|
||||
("https://e926.net/posts?tags=anry", {
|
||||
"url": "12198b275c62ffe2de67cca676c8e64de80c425d",
|
||||
"content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58",
|
||||
}),
|
||||
("https://e926.net/post/index/1/anry"),
|
||||
("https://e926.net/post?tags=anry"),
|
||||
)
|
||||
|
||||
|
||||
class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor):
|
||||
"""Extractor for e621 pools"""
|
||||
pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)"
|
||||
test = (
|
||||
("https://e621.net/pools/73", {
|
||||
"url": "1bd09a72715286a79eea3b7f09f51b3493eb579a",
|
||||
"content": "91abe5d5334425d9787811d7f06d34c77974cd22",
|
||||
}),
|
||||
("https://e621.net/pool/show/73"),
|
||||
|
||||
("https://e926.net/pools/73", {
|
||||
"url": "6936f1b6a18c5c25bee7cad700088dbc2503481b",
|
||||
"content": "91abe5d5334425d9787811d7f06d34c77974cd22",
|
||||
}),
|
||||
("https://e926.net/pool/show/73"),
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
self.log.info("Fetching posts of pool %s", self.pool_id)
|
||||
|
||||
id_to_post = {
|
||||
post["id"]: post
|
||||
for post in self._pagination(
|
||||
"/posts.json", {"tags": "pool:" + self.pool_id})
|
||||
}
|
||||
|
||||
posts = []
|
||||
append = posts.append
|
||||
for num, pid in enumerate(self.post_ids, 1):
|
||||
if pid in id_to_post:
|
||||
post = id_to_post[pid]
|
||||
post["num"] = num
|
||||
append(post)
|
||||
else:
|
||||
self.log.warning("Post %s is unavailable", pid)
|
||||
return posts
|
||||
|
||||
|
||||
class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor):
|
||||
"""Extractor for single e621 posts"""
|
||||
pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)"
|
||||
test = (
|
||||
("https://e621.net/posts/535", {
|
||||
"url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529",
|
||||
"content": "66f46e96a893fba8e694c4e049b23c2acc9af462",
|
||||
}),
|
||||
("https://e621.net/post/show/535"),
|
||||
|
||||
("https://e926.net/posts/535", {
|
||||
"url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf",
|
||||
"content": "66f46e96a893fba8e694c4e049b23c2acc9af462",
|
||||
}),
|
||||
("https://e926.net/post/show/535"),
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
url = "{}/posts/{}.json".format(self.root, self.post_id)
|
||||
return (self.request(url).json()["post"],)
|
||||
|
||||
|
||||
class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor):
|
||||
"""Extractor for popular images from e621"""
|
||||
pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?"
|
||||
test = (
|
||||
("https://e621.net/explore/posts/popular"),
|
||||
(("https://e621.net/explore/posts/popular"
|
||||
"?date=2019-06-01&scale=month"), {
|
||||
"pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+",
|
||||
"count": ">= 70",
|
||||
}),
|
||||
|
||||
("https://e926.net/explore/posts/popular"),
|
||||
(("https://e926.net/explore/posts/popular"
|
||||
"?date=2019-06-01&scale=month"), {
|
||||
"pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+",
|
||||
"count": ">= 70",
|
||||
}),
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
if self.page_start is None:
|
||||
self.page_start = 1
|
||||
return self._pagination("/popular.json", self.params, True)
|
||||
|
||||
|
||||
class E621FavoriteExtractor(E621Extractor):
|
||||
"""Extractor for e621 favorites"""
|
||||
subcategory = "favorite"
|
||||
directory_fmt = ("{category}", "Favorites", "{user_id}")
|
||||
archive_fmt = "f_{user_id}_{id}"
|
||||
pattern = BASE_PATTERN + r"/favorites(?:\?([^#]*))?"
|
||||
test = (
|
||||
("https://e621.net/favorites"),
|
||||
("https://e621.net/favorites?page=2&user_id=53275", {
|
||||
"pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+",
|
||||
"count": "> 260",
|
||||
}),
|
||||
|
||||
("https://e926.net/favorites"),
|
||||
("https://e926.net/favorites?page=2&user_id=53275", {
|
||||
"pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+",
|
||||
"count": "> 260",
|
||||
}),
|
||||
)
|
||||
|
||||
def __init__(self, match):
|
||||
E621Extractor.__init__(self, match)
|
||||
self.query = text.parse_query(match.group(match.lastindex))
|
||||
|
||||
def metadata(self):
|
||||
return {"user_id": self.query.get("user_id", "")}
|
||||
|
||||
def posts(self):
|
||||
if self.page_start is None:
|
||||
self.page_start = 1
|
||||
return self._pagination("/favorites.json", self.query, True)
|
||||
Loading…
Reference in new issue