You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
308 lines
10 KiB
308 lines
10 KiB
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2019-2023 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extractors for https://www.tsumino.com/"""
|
|
|
|
from .common import GalleryExtractor, Extractor, Message
|
|
from .. import text, exception
|
|
from ..cache import cache
|
|
|
|
|
|
class TsuminoBase():
|
|
"""Base class for tsumino extractors"""
|
|
category = "tsumino"
|
|
cookies_domain = "www.tsumino.com"
|
|
root = "https://www.tsumino.com"
|
|
|
|
def login(self):
|
|
username, password = self._get_auth_info()
|
|
if username:
|
|
self.cookies_update(self._login_impl(username, password))
|
|
else:
|
|
self.cookies.setdefault(
|
|
"ASP.NET_SessionId", "x1drgggilez4cpkttneukrc5")
|
|
|
|
@cache(maxage=14*86400, keyarg=1)
|
|
def _login_impl(self, username, password):
|
|
self.log.info("Logging in as %s", username)
|
|
url = "{}/Account/Login".format(self.root)
|
|
headers = {"Referer": url}
|
|
data = {"Username": username, "Password": password}
|
|
|
|
response = self.request(url, method="POST", headers=headers, data=data)
|
|
if not response.history:
|
|
raise exception.AuthenticationError()
|
|
return self.cookies
|
|
|
|
|
|
class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor):
|
|
"""Extractor for image galleries on tsumino.com"""
|
|
pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com"
|
|
r"/(?:entry|Book/Info|Read/(?:Index|View))/(\d+)")
|
|
example = "https://www.tsumino.com/entry/12345"
|
|
|
|
def __init__(self, match):
|
|
self.gallery_id = match.group(1)
|
|
url = "{}/entry/{}".format(self.root, self.gallery_id)
|
|
GalleryExtractor.__init__(self, match, url)
|
|
|
|
def metadata(self, page):
|
|
extr = text.extract_from(page)
|
|
title = extr('"og:title" content="', '"')
|
|
title_en, _, title_jp = text.unescape(title).partition("/")
|
|
title_en = title_en.strip()
|
|
title_jp = title_jp.strip()
|
|
|
|
return {
|
|
"gallery_id": text.parse_int(self.gallery_id),
|
|
"title" : title_en or title_jp,
|
|
"title_en" : title_en,
|
|
"title_jp" : title_jp,
|
|
"thumbnail" : extr('"og:image" content="', '"'),
|
|
"uploader" : text.remove_html(extr('id="Uploader">', '</div>')),
|
|
"date" : text.parse_datetime(
|
|
extr('id="Uploaded">', '</div>').strip(), "%Y %B %d"),
|
|
"rating" : text.parse_float(extr(
|
|
'id="Rating">', '</div>').partition(" ")[0]),
|
|
"type" : text.remove_html(extr('id="Category">' , '</div>')),
|
|
"collection": text.remove_html(extr('id="Collection">', '</div>')),
|
|
"group" : text.split_html(extr('id="Group">' , '</div>')),
|
|
"artist" : text.split_html(extr('id="Artist">' , '</div>')),
|
|
"parody" : text.split_html(extr('id="Parody">' , '</div>')),
|
|
"characters": text.split_html(extr('id="Character">' , '</div>')),
|
|
"tags" : text.split_html(extr('id="Tag">' , '</div>')),
|
|
"language" : "English",
|
|
"lang" : "en",
|
|
}
|
|
|
|
def images(self, page):
|
|
url = "{}/Read/Index/{}?page=1".format(self.root, self.gallery_id)
|
|
headers = {"Referer": self.gallery_url}
|
|
response = self.request(url, headers=headers, fatal=False)
|
|
|
|
if "/Auth/" in response.url:
|
|
raise exception.StopExtraction(
|
|
"Failed to get gallery JSON data. Visit '%s' in a browser "
|
|
"and solve the CAPTCHA to continue.", response.url)
|
|
|
|
page = response.text
|
|
tpl, pos = text.extract(page, 'data-cdn="', '"')
|
|
cnt, pos = text.extract(page, '> of ', '<', pos)
|
|
base, _, params = text.unescape(tpl).partition("[PAGE]")
|
|
|
|
return [
|
|
(base + str(i) + params, None)
|
|
for i in range(1, text.parse_int(cnt)+1)
|
|
]
|
|
|
|
|
|
class TsuminoSearchExtractor(TsuminoBase, Extractor):
|
|
"""Extractor for search results on tsumino.com"""
|
|
subcategory = "search"
|
|
pattern = r"(?i)(?:https?://)?(?:www\.)?tsumino\.com/(?:Books/?)?#(.+)"
|
|
example = "https://www.tsumino.com/Books#QUERY"
|
|
|
|
def __init__(self, match):
|
|
Extractor.__init__(self, match)
|
|
self.query = match.group(1)
|
|
|
|
def items(self):
|
|
for gallery in self.galleries():
|
|
url = "{}/entry/{}".format(self.root, gallery["id"])
|
|
gallery["_extractor"] = TsuminoGalleryExtractor
|
|
yield Message.Queue, url, gallery
|
|
|
|
def galleries(self):
|
|
"""Return all gallery results matching 'self.query'"""
|
|
url = "{}/Search/Operate?type=Book".format(self.root)
|
|
headers = {
|
|
"Referer": "{}/".format(self.root),
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
}
|
|
data = {
|
|
"PageNumber": 1,
|
|
"Text": "",
|
|
"Sort": "Newest",
|
|
"List": "0",
|
|
"Length": "0",
|
|
"MinimumRating": "0",
|
|
"ExcludeList": "0",
|
|
"CompletelyExcludeHated": "false",
|
|
}
|
|
data.update(self._parse(self.query))
|
|
|
|
while True:
|
|
info = self.request(
|
|
url, method="POST", headers=headers, data=data).json()
|
|
|
|
for gallery in info["data"]:
|
|
yield gallery["entry"]
|
|
|
|
if info["pageNumber"] >= info["pageCount"]:
|
|
return
|
|
data["PageNumber"] += 1
|
|
|
|
def _parse(self, query):
|
|
try:
|
|
if query.startswith("?"):
|
|
return self._parse_simple(query)
|
|
return self._parse_jsurl(query)
|
|
except Exception as exc:
|
|
raise exception.StopExtraction(
|
|
"Invalid search query '%s' (%s)", query, exc)
|
|
|
|
@staticmethod
|
|
def _parse_simple(query):
|
|
"""Parse search query with format '?<key>=value>'"""
|
|
key, _, value = query.partition("=")
|
|
tag_types = {
|
|
"Tag": "1",
|
|
"Category": "2",
|
|
"Collection": "3",
|
|
"Group": "4",
|
|
"Artist": "5",
|
|
"Parody": "6",
|
|
"Character": "7",
|
|
"Uploader": "100",
|
|
}
|
|
|
|
return {
|
|
"Tags[0][Type]": tag_types[key[1:].capitalize()],
|
|
"Tags[0][Text]": text.unquote(value).replace("+", " "),
|
|
"Tags[0][Exclude]": "false",
|
|
}
|
|
|
|
@staticmethod
|
|
def _parse_jsurl(data):
|
|
"""Parse search query in JSURL format
|
|
|
|
Nested lists and dicts are handled in a special way to deal
|
|
with the way Tsumino expects its parameters -> expand(...)
|
|
|
|
Example: ~(name~'John*20Doe~age~42~children~(~'Mary~'Bill))
|
|
Ref: https://github.com/Sage/jsurl
|
|
"""
|
|
if not data:
|
|
return {}
|
|
i = 0
|
|
imax = len(data)
|
|
|
|
def eat(expected):
|
|
nonlocal i
|
|
|
|
if data[i] != expected:
|
|
error = "bad JSURL syntax: expected '{}', got {}".format(
|
|
expected, data[i])
|
|
raise ValueError(error)
|
|
i += 1
|
|
|
|
def decode():
|
|
nonlocal i
|
|
|
|
beg = i
|
|
result = ""
|
|
|
|
while i < imax:
|
|
ch = data[i]
|
|
|
|
if ch not in "~)*!":
|
|
i += 1
|
|
|
|
elif ch == "*":
|
|
if beg < i:
|
|
result += data[beg:i]
|
|
if data[i + 1] == "*":
|
|
result += chr(int(data[i+2:i+6], 16))
|
|
i += 6
|
|
else:
|
|
result += chr(int(data[i+1:i+3], 16))
|
|
i += 3
|
|
beg = i
|
|
|
|
elif ch == "!":
|
|
if beg < i:
|
|
result += data[beg:i]
|
|
result += "$"
|
|
i += 1
|
|
beg = i
|
|
|
|
else:
|
|
break
|
|
|
|
return result + data[beg:i]
|
|
|
|
def parse_one():
|
|
nonlocal i
|
|
|
|
eat('~')
|
|
result = ""
|
|
ch = data[i]
|
|
|
|
if ch == "(":
|
|
i += 1
|
|
|
|
if data[i] == "~":
|
|
result = []
|
|
if data[i+1] == ")":
|
|
i += 1
|
|
else:
|
|
result.append(parse_one())
|
|
while data[i] == "~":
|
|
result.append(parse_one())
|
|
|
|
else:
|
|
result = {}
|
|
|
|
if data[i] != ")":
|
|
while True:
|
|
key = decode()
|
|
value = parse_one()
|
|
for ekey, evalue in expand(key, value):
|
|
result[ekey] = evalue
|
|
if data[i] != "~":
|
|
break
|
|
i += 1
|
|
eat(")")
|
|
|
|
elif ch == "'":
|
|
i += 1
|
|
result = decode()
|
|
|
|
else:
|
|
beg = i
|
|
i += 1
|
|
|
|
while i < imax and data[i] not in "~)":
|
|
i += 1
|
|
|
|
sub = data[beg:i]
|
|
if ch in "0123456789-":
|
|
fval = float(sub)
|
|
ival = int(fval)
|
|
result = ival if ival == fval else fval
|
|
else:
|
|
if sub not in ("true", "false", "null"):
|
|
raise ValueError("bad value keyword: " + sub)
|
|
result = sub
|
|
|
|
return result
|
|
|
|
def expand(key, value):
|
|
if isinstance(value, list):
|
|
for index, cvalue in enumerate(value):
|
|
ckey = "{}[{}]".format(key, index)
|
|
yield from expand(ckey, cvalue)
|
|
elif isinstance(value, dict):
|
|
for ckey, cvalue in value.items():
|
|
ckey = "{}[{}]".format(key, ckey)
|
|
yield from expand(ckey, cvalue)
|
|
else:
|
|
yield key, value
|
|
|
|
return parse_one()
|