You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
415 lines
15 KiB
415 lines
15 KiB
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2014-2019 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extract images from galleries at https://exhentai.org/"""
|
|
|
|
from .common import Extractor, Message
|
|
from .. import text, util, exception
|
|
from ..cache import cache
|
|
import itertools
|
|
import random
|
|
import time
|
|
import math
|
|
|
|
|
|
BASE_PATTERN = r"(?:https?://)?(e[x-]|g\.e-)hentai\.org"
|
|
|
|
|
|
class ExhentaiExtractor(Extractor):
|
|
"""Base class for exhentai extractors"""
|
|
category = "exhentai"
|
|
directory_fmt = ("{category}", "{gallery_id}")
|
|
filename_fmt = (
|
|
"{gallery_id}_{num:>04}_{image_token}_{filename}.{extension}")
|
|
archive_fmt = "{gallery_id}_{num}"
|
|
cookiedomain = ".exhentai.org"
|
|
cookienames = ("ipb_member_id", "ipb_pass_hash")
|
|
root = "https://exhentai.org"
|
|
|
|
def __init__(self, match):
|
|
Extractor.__init__(self, match)
|
|
self.limits = self.config("limits", True)
|
|
self.original = self.config("original", True)
|
|
self.wait_min = self.config("wait-min", 3)
|
|
self.wait_max = self.config("wait-max", 6)
|
|
|
|
self._remaining = 0
|
|
if self.wait_max < self.wait_min:
|
|
self.wait_max = self.wait_min
|
|
self.session.headers["Referer"] = self.root + "/"
|
|
|
|
def request(self, *args, **kwargs):
|
|
response = Extractor.request(self, *args, **kwargs)
|
|
if self._is_sadpanda(response):
|
|
self.log.info("sadpanda.jpg")
|
|
raise exception.AuthorizationError()
|
|
return response
|
|
|
|
def wait(self, waittime=None):
|
|
"""Wait for a randomly chosen amount of seconds"""
|
|
if not waittime:
|
|
waittime = random.uniform(self.wait_min, self.wait_max)
|
|
else:
|
|
waittime = random.uniform(waittime * 0.66, waittime * 1.33)
|
|
time.sleep(waittime)
|
|
|
|
def login(self):
|
|
"""Login and set necessary cookies"""
|
|
if self._check_cookies(self.cookienames):
|
|
return
|
|
username, password = self._get_auth_info()
|
|
if username:
|
|
self._update_cookies(self._login_impl(username, password))
|
|
else:
|
|
self.log.info("no username given; using e-hentai.org")
|
|
self.root = "https://e-hentai.org"
|
|
self.original = False
|
|
self.limits = False
|
|
self.session.cookies["nw"] = "1"
|
|
|
|
@cache(maxage=90*24*3600, keyarg=1)
|
|
def _login_impl(self, username, password):
|
|
self.log.info("Logging in as %s", username)
|
|
url = "https://forums.e-hentai.org/index.php?act=Login&CODE=01"
|
|
headers = {
|
|
"Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1",
|
|
}
|
|
data = {
|
|
"CookieDate": "1",
|
|
"b": "d",
|
|
"bt": "1-1",
|
|
"UserName": username,
|
|
"PassWord": password,
|
|
"ipb_login_submit": "Login!",
|
|
}
|
|
|
|
response = self.request(url, method="POST", headers=headers, data=data)
|
|
if "You are now logged in as:" not in response.text:
|
|
raise exception.AuthenticationError()
|
|
return {c: response.cookies[c] for c in self.cookienames}
|
|
|
|
@staticmethod
|
|
def _is_sadpanda(response):
|
|
"""Return True if the response object contains a sad panda"""
|
|
return (
|
|
response.headers.get("Content-Length") == "9615" and
|
|
"sadpanda.jpg" in response.headers.get("Content-Disposition", "")
|
|
)
|
|
|
|
|
|
class ExhentaiGalleryExtractor(ExhentaiExtractor):
|
|
"""Extractor for image galleries from exhentai.org"""
|
|
subcategory = "gallery"
|
|
pattern = (BASE_PATTERN +
|
|
r"(?:/g/(\d+)/([\da-f]{10})"
|
|
r"|/s/([\da-f]{10})/(\d+)-(\d+))")
|
|
test = (
|
|
("https://exhentai.org/g/960460/4f0e369d82/", {
|
|
"keyword": "993bfaf68b4823084fbd0d3339564666463b1432",
|
|
"content": "493d759de534355c9f55f8e365565b62411de146",
|
|
}),
|
|
("https://exhentai.org/g/960461/4f0e369d82/", {
|
|
"exception": exception.NotFoundError,
|
|
}),
|
|
("http://exhentai.org/g/962698/7f02358e00/", {
|
|
"exception": exception.AuthorizationError,
|
|
}),
|
|
("https://exhentai.org/s/3957343c3b/960460-5", {
|
|
"count": 2,
|
|
}),
|
|
("https://e-hentai.org/g/960460/4f0e369d82/"),
|
|
("https://g.e-hentai.org/g/960460/4f0e369d82/"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
ExhentaiExtractor.__init__(self, match)
|
|
self.key = {}
|
|
self.count = 0
|
|
self.gallery_id = text.parse_int(match.group(2) or match.group(5))
|
|
self.gallery_token = match.group(3)
|
|
self.image_token = match.group(4)
|
|
self.image_num = text.parse_int(match.group(6), 1)
|
|
|
|
def items(self):
|
|
self.login()
|
|
|
|
if self.gallery_token:
|
|
gpage = self._gallery_page()
|
|
self.image_token = text.extract(gpage, 'hentai.org/s/', '"')[0]
|
|
self.wait()
|
|
ipage = self._image_page()
|
|
else:
|
|
ipage = self._image_page()
|
|
part = text.extract(ipage, 'hentai.org/g/', '"')[0]
|
|
self.gallery_token = part.split("/")[1]
|
|
self.wait()
|
|
gpage = self._gallery_page()
|
|
|
|
data = self.get_metadata(gpage)
|
|
self.count = data["count"]
|
|
|
|
yield Message.Version, 1
|
|
yield Message.Directory, data
|
|
|
|
images = itertools.chain(
|
|
(self.image_from_page(ipage),), self.images_from_api())
|
|
for url, image in images:
|
|
data.update(image)
|
|
if self.limits:
|
|
self._check_limits(data)
|
|
if "/fullimg.php" in url:
|
|
data["extension"] = ""
|
|
self.wait(1.5)
|
|
yield Message.Url, url, data
|
|
|
|
def get_metadata(self, page):
|
|
"""Extract gallery metadata"""
|
|
data, pos = text.extract_all(page, (
|
|
("title" , '<h1 id="gn">', '</h1>'),
|
|
("title_jp" , '<h1 id="gj">', '</h1>'),
|
|
("date" , '>Posted:</td><td class="gdt2">', '</td>'),
|
|
("parent" , '>Parent:</td><td class="gdt2"><a href="', '"'),
|
|
("visible" , '>Visible:</td><td class="gdt2">', '<'),
|
|
("language" , '>Language:</td><td class="gdt2">', ' '),
|
|
("gallery_size", '>File Size:</td><td class="gdt2">', '<'),
|
|
("count" , '>Length:</td><td class="gdt2">', ' '),
|
|
))
|
|
|
|
data["lang"] = util.language_to_code(data["language"])
|
|
data["title"] = text.unescape(data["title"])
|
|
data["title_jp"] = text.unescape(data["title_jp"])
|
|
data["count"] = text.parse_int(data["count"])
|
|
data["gallery_id"] = self.gallery_id
|
|
data["gallery_token"] = self.gallery_token
|
|
data["gallery_size"] = text.parse_bytes(
|
|
data["gallery_size"].rstrip("Bb"))
|
|
data["tags"] = [
|
|
text.unquote(tag)
|
|
for tag in text.extract_iter(page, 'hentai.org/tag/', '"', pos)
|
|
]
|
|
return data
|
|
|
|
def image_from_page(self, page):
|
|
"""Get image url and data from webpage"""
|
|
info = text.extract_all(page, (
|
|
(None , '<div id="i3"><a onclick="return load_image(', ''),
|
|
("nextkey" , "'", "'"),
|
|
("url" , '<img id="img" src="', '"'),
|
|
("origurl" , 'hentai.org/fullimg.php', '"'),
|
|
("originfo", 'ownload original', '<'),
|
|
("startkey", 'var startkey="', '";'),
|
|
("showkey" , 'var showkey="', '";'),
|
|
))[0]
|
|
self.key["start"] = info["startkey"]
|
|
self.key["show"] = info["showkey"]
|
|
self.key["next"] = info["nextkey"]
|
|
|
|
if self.original and info["origurl"]:
|
|
part = text.unescape(info["origurl"])
|
|
url = self.root + "/fullimg.php" + part
|
|
data = self._parse_original_info(info["originfo"])
|
|
else:
|
|
url = info["url"]
|
|
data = self._parse_image_info(url)
|
|
|
|
data["num"] = self.image_num
|
|
data["image_token"] = info["startkey"]
|
|
return url, text.nameext_from_url(info["url"], data)
|
|
|
|
def images_from_api(self):
|
|
"""Get image url and data from api calls"""
|
|
api_url = self.root + "/api.php"
|
|
nextkey = self.key["next"]
|
|
request = {
|
|
"method" : "showpage",
|
|
"gid" : self.gallery_id,
|
|
"imgkey" : nextkey,
|
|
"showkey": self.key["show"],
|
|
}
|
|
for request["page"] in range(self.image_num + 1, self.count + 1):
|
|
self.wait()
|
|
page = self.request(api_url, method="POST", json=request).json()
|
|
imgkey = nextkey
|
|
nextkey, pos = text.extract(page["i3"], "'", "'")
|
|
imgurl , pos = text.extract(page["i3"], 'id="img" src="', '"', pos)
|
|
origurl, pos = text.extract(page["i7"], '<a href="', '"')
|
|
|
|
if self.original and origurl:
|
|
url = text.unescape(origurl)
|
|
data = self._parse_original_info(
|
|
text.extract(page["i7"], "ownload original", "<", pos)[0])
|
|
else:
|
|
url = imgurl
|
|
data = self._parse_image_info(url)
|
|
|
|
data["num"] = request["page"]
|
|
data["image_token"] = imgkey
|
|
yield url, text.nameext_from_url(imgurl, data)
|
|
|
|
request["imgkey"] = nextkey
|
|
|
|
def _gallery_page(self):
|
|
url = "{}/g/{}/{}/".format(
|
|
self.root, self.gallery_id, self.gallery_token)
|
|
response = self.request(url, expect=range(400, 500))
|
|
page = response.text
|
|
|
|
if response.status_code == 404 and "Gallery Not Available" in page:
|
|
raise exception.AuthorizationError()
|
|
if page.startswith(("Key missing", "Gallery not found")):
|
|
raise exception.NotFoundError("gallery")
|
|
return page
|
|
|
|
def _image_page(self):
|
|
url = "{}/s/{}/{}-{}".format(
|
|
self.root, self.image_token, self.gallery_id, self.image_num)
|
|
page = self.request(url, expect=range(400, 500)).text
|
|
|
|
if page.startswith(("Invalid page", "Keep trying")):
|
|
raise exception.NotFoundError("image page")
|
|
return page
|
|
|
|
def _check_limits(self, data):
|
|
if not self._remaining or data["num"] % 20 == 0:
|
|
self._update_limits()
|
|
self._remaining -= data["cost"]
|
|
|
|
if self._remaining <= 0:
|
|
url = "{}/s/{}/{}-{}".format(
|
|
self.root, data["image_token"], self.gallery_id, data["num"])
|
|
self.log.error(
|
|
"Image limit reached! Reset it and continue with "
|
|
"'%s' as URL.", url)
|
|
raise exception.StopExtraction()
|
|
|
|
def _update_limits(self):
|
|
url = "https://e-hentai.org/home.php"
|
|
cookies = {
|
|
cookie.name: cookie.value
|
|
for cookie in self.session.cookies
|
|
if cookie.domain == self.cookiedomain and cookie.name != "igneous"
|
|
}
|
|
|
|
page = self.request(url, cookies=cookies).text
|
|
current, pos = text.extract(page, "<strong>", "</strong>")
|
|
maximum, pos = text.extract(page, "<strong>", "</strong>", pos)
|
|
self._remaining = text.parse_int(maximum) - text.parse_int(current)
|
|
|
|
@staticmethod
|
|
def _parse_image_info(url):
|
|
parts = url.split("/")[4].split("-")
|
|
return {
|
|
"width": text.parse_int(parts[2]),
|
|
"height": text.parse_int(parts[3]),
|
|
"size": text.parse_int(parts[1]),
|
|
"cost": 1,
|
|
}
|
|
|
|
@staticmethod
|
|
def _parse_original_info(info):
|
|
parts = info.lstrip().split(" ")
|
|
size = text.parse_bytes(parts[3] + parts[4][0])
|
|
return {
|
|
"width": text.parse_int(parts[0]),
|
|
"height": text.parse_int(parts[2]),
|
|
"size": size,
|
|
"cost": 1 + math.ceil(size * 5 / 1024 / 1024)
|
|
}
|
|
|
|
|
|
class ExhentaiSearchExtractor(ExhentaiExtractor):
|
|
"""Extractor for exhentai search results"""
|
|
subcategory = "search"
|
|
pattern = BASE_PATTERN + r"/?\?(.*)$"
|
|
test = (
|
|
("https://exhentai.org/?f_search=touhou"),
|
|
(("https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0"
|
|
"&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0"
|
|
"&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter"), {
|
|
"pattern": ExhentaiGalleryExtractor.pattern,
|
|
"range": "1-30",
|
|
"count": 30,
|
|
}),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
ExhentaiExtractor.__init__(self, match)
|
|
self.params = text.parse_query(match.group(2) or "")
|
|
self.params["page"] = text.parse_int(self.params.get("page"))
|
|
self.search_url = self.root
|
|
|
|
def items(self):
|
|
self.login()
|
|
self.init()
|
|
yield Message.Version, 1
|
|
|
|
while True:
|
|
page = self.request(self.search_url, params=self.params).text
|
|
|
|
for row in text.extract_iter(page, '<tr class="gtr', '</tr>'):
|
|
yield self._parse_row(row)
|
|
|
|
if 'class="ptdd">><' in page or ">No hits found</p>" in page:
|
|
return
|
|
self.params["page"] += 1
|
|
self.wait()
|
|
|
|
def init(self):
|
|
pass
|
|
|
|
def _parse_row(self, row, extr=text.extract):
|
|
"""Parse information of a single result row"""
|
|
gtype, pos = extr(row, ' alt="', '"')
|
|
date , pos = extr(row, 'nowrap">', '<', pos)
|
|
url , pos = extr(row, ' class="it5"><a href="', '"', pos)
|
|
title, pos = extr(row, '>', '<', pos)
|
|
key , last = self._parse_last(row, pos)
|
|
parts = url.rsplit("/", 3)
|
|
|
|
return Message.Queue, url, {
|
|
"type": gtype,
|
|
"date": date,
|
|
"gallery_id": text.parse_int(parts[1]),
|
|
"gallery_token": parts[2],
|
|
"title": text.unescape(title),
|
|
"_extractor": ExhentaiGalleryExtractor,
|
|
key: last,
|
|
}
|
|
|
|
def _parse_last(self, row, pos):
|
|
"""Parse the last column of a result row"""
|
|
return "uploader", text.remove_html(
|
|
text.extract(row, '<td class="itu">', '</td>', pos)[0])
|
|
|
|
|
|
class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor):
|
|
"""Extractor for favorited exhentai galleries"""
|
|
subcategory = "favorite"
|
|
pattern = BASE_PATTERN + r"/favorites\.php(?:\?(.*))?"
|
|
test = (
|
|
("https://exhentai.org/favorites.php"),
|
|
("https://exhentai.org/favorites.php?favcat=1&f_search=touhou"
|
|
"&f_apply=Search+Favorites"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
ExhentaiSearchExtractor.__init__(self, match)
|
|
self.search_url = self.root + "/favorites.php"
|
|
|
|
def init(self):
|
|
# The first request to '/favorites.php' will return an empty list
|
|
# if the 's' cookie isn't set (maybe on some other conditions as well),
|
|
# so we make a "noop" request to get all the correct cookie values
|
|
# and to get a filled favorite list on the next one.
|
|
# TODO: proper cookie storage
|
|
self.request(self.url)
|
|
self.wait(1.5)
|
|
|
|
def _parse_last(self, row, pos):
|
|
return "date_favorited", text.extract(row, 'nowrap">', '<', pos)[0]
|