', '<'),
"uploader" : extr('
', '
'),
"date" : text.parse_datetime(extr(
'>Posted:
', ' | '), "%Y-%m-%d %H:%M"),
"parent" : extr(
'>Parent:
Visible: | ', '<'),
"language" : extr('>Language: | ', ' '),
"filesize" : text.parse_bytes(extr(
'>File Size: | ', '<').rstrip("Bbi")),
"filecount" : extr('>Length: | ', ' '),
"favorites" : extr('id="favcount">', ' '),
"rating" : extr(">Average: ", "<"),
"torrentcount" : extr('>Torrent Download (', ')'),
}
if data["uploader"].startswith("<"):
data["uploader"] = text.unescape(text.extr(
data["uploader"], ">", "<"))
f = data["favorites"][0]
if f == "N":
data["favorites"] = "0"
elif f == "O":
data["favorites"] = "1"
data["lang"] = util.language_to_code(data["language"])
data["tags"] = [
text.unquote(tag.replace("+", " "))
for tag in text.extract_iter(page, 'hentai.org/tag/', '"')
]
return data
def metadata_from_api(self):
data = {
"method" : "gdata",
"gidlist" : ((self.gallery_id, self.gallery_token),),
"namespace": 1,
}
data = self.request(self.api_url, method="POST", json=data).json()
if "error" in data:
raise exception.StopExtraction(data["error"])
return data["gmetadata"][0]
def image_from_page(self, page):
"""Get image url and data from webpage"""
pos = page.index('= 0:
origurl, pos = text.rextract(i6, '"', '"', pos)
url = text.unescape(origurl)
data = self._parse_original_info(text.extract(
i6, "ownload original", "<", pos)[0])
data["_fallback"] = self._fallback_original(nl, url)
else:
url = imgurl
data = self._parse_image_info(url)
data["_fallback"] = self._fallback_1280(
nl, request["page"], imgkey)
except IndexError:
self.log.debug("Page content:\n%s", page)
raise exception.StopExtraction(
"Unable to parse image info for '%s'", url)
data["num"] = request["page"]
data["image_token"] = imgkey
data["_url_1280"] = imgurl
data["_nl"] = nl
self._check_509(imgurl)
yield url, text.nameext_from_url(url, data)
request["imgkey"] = nextkey
def _validate_response(self, response):
if not response.history and response.headers.get(
"content-type", "").startswith("text/html"):
page = response.text
self.log.warning("'%s'", page)
if " requires GP" in page:
gp = self.config("gp")
if gp == "stop":
raise exception.StopExtraction("Not enough GP")
elif gp == "wait":
input("Press ENTER to continue.")
return response.url
self.log.info("Falling back to non-original downloads")
self.original = False
return self.data["_url_1280"]
self._report_limits()
return True
def _report_limits(self):
ExhentaiExtractor.LIMIT = True
raise exception.StopExtraction("Image limit reached!")
def _check_limits(self, data):
if not self._remaining or data["num"] % 25 == 0:
self._update_limits()
self._remaining -= data["cost"]
if self._remaining <= 0:
self._report_limits()
def _check_509(self, url):
# full 509.gif URLs
# - https://exhentai.org/img/509.gif
# - https://ehgt.org/g/509.gif
if url.endswith(("hentai.org/img/509.gif",
"ehgt.org/g/509.gif")):
self.log.debug(url)
self._report_limits()
def _update_limits(self):
url = "https://e-hentai.org/home.php"
cookies = {
cookie.name: cookie.value
for cookie in self.cookies
if cookie.domain == self.cookies_domain and
cookie.name != "igneous"
}
page = self.request(url, cookies=cookies).text
current = text.extr(page, "", "")
self.log.debug("Image Limits: %s/%s", current, self.limits)
self._remaining = self.limits - text.parse_int(current)
def _gallery_page(self):
url = "{}/g/{}/{}/".format(
self.root, self.gallery_id, self.gallery_token)
response = self.request(url, fatal=False)
page = response.text
if response.status_code == 404 and "Gallery Not Available" in page:
raise exception.AuthorizationError()
if page.startswith(("Key missing", "Gallery not found")):
raise exception.NotFoundError("gallery")
if "hentai.org/mpv/" in page:
self.log.warning("Enabled Multi-Page Viewer is not supported")
return page
def _image_page(self):
url = "{}/s/{}/{}-{}".format(
self.root, self.image_token, self.gallery_id, self.image_num)
page = self.request(url, fatal=False).text
if page.startswith(("Invalid page", "Keep trying")):
raise exception.NotFoundError("image page")
return page
def _fallback_original(self, nl, fullimg):
url = "{}?nl={}".format(fullimg, nl)
for _ in util.repeat(self.fallback_retries):
yield url
def _fallback_1280(self, nl, num, token=None):
if not token:
token = self.key_start
for _ in util.repeat(self.fallback_retries):
url = "{}/s/{}/{}-{}?nl={}".format(
self.root, token, self.gallery_id, num, nl)
page = self.request(url, fatal=False).text
if page.startswith(("Invalid page", "Keep trying")):
return
url, data = self.image_from_page(page)
yield url
nl = data["_nl"]
@staticmethod
def _parse_image_info(url):
for part in url.split("/")[4:]:
try:
_, size, width, height, _ = part.split("-")
break
except ValueError:
pass
else:
size = width = height = 0
return {
"cost" : 1,
"size" : text.parse_int(size),
"width" : text.parse_int(width),
"height": text.parse_int(height),
}
@staticmethod
def _parse_original_info(info):
parts = info.lstrip().split(" ")
size = text.parse_bytes(parts[3] + parts[4][0])
return {
# 1 initial point + 1 per 0.1 MB
"cost" : 1 + math.ceil(size / 100000),
"size" : size,
"width" : text.parse_int(parts[0]),
"height": text.parse_int(parts[2]),
}
class ExhentaiSearchExtractor(ExhentaiExtractor):
"""Extractor for exhentai search results"""
subcategory = "search"
pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))"
example = "https://e-hentai.org/?f_search=QUERY"
def __init__(self, match):
ExhentaiExtractor.__init__(self, match)
_, query, tag = match.groups()
if tag:
if "+" in tag:
ns, _, tag = tag.rpartition(":")
tag = '{}:"{}$"'.format(ns, tag.replace("+", " "))
else:
tag += "$"
self.params = {"f_search": tag, "page": 0}
else:
self.params = text.parse_query(query)
if "next" not in self.params:
self.params["page"] = text.parse_int(self.params.get("page"))
def _init(self):
self.search_url = self.root
def items(self):
self.login()
data = {"_extractor": ExhentaiGalleryExtractor}
search_url = self.search_url
params = self.params
while True:
last = None
page = self.request(search_url, params=params).text
for gallery in ExhentaiGalleryExtractor.pattern.finditer(page):
url = gallery.group(0)
if url == last:
continue
last = url
data["gallery_id"] = text.parse_int(gallery.group(2))
data["gallery_token"] = gallery.group(3)
yield Message.Queue, url + "/", data
next_url = text.extr(page, 'nexturl="', '"', None)
if next_url is not None:
if not next_url:
return
search_url = next_url
params = None
elif 'class="ptdd">><' in page or ">No hits found" in page:
return
else:
params["page"] += 1
class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor):
"""Extractor for favorited exhentai galleries"""
subcategory = "favorite"
pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?"
example = "https://e-hentai.org/favorites.php"
def _init(self):
self.search_url = self.root + "/favorites.php"
|