mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[pixiv] update to new extractor interface

Browse Source
pull/13/head
Mike Fährmann 10 years ago
parent 7c8d787077
commit 1cd25b5369
2 changed files with 116 additions and 85 deletions
Show Stats Download Patch File Download Diff File

4
gallery_dl/extractor/common.py
Unescape View File

@ -69,11 +69,11 @@ class SequentialExtractor(Extractor):
Extractor.__init__(self)
class AsyncExtractor(Extractor):
class AsynchronousExtractor(Extractor):
def __init__(self, config):
Extractor.__init__(self)
queue_size = int(config.get("queue-size", 5))
queue_size = int(config.get("general", "queue-size", fallback=5))
self.__queue = queue.Queue(maxsize=queue_size)
self.__thread = threading.Thread(target=self.async_items)
# self.__thread = threading.Thread(target=self.async_images, daemon=True)

197
gallery_dl/extractor/pixiv.py
Unescape View File

@ -1,94 +1,116 @@
from .common import AsyncExtractor
from ..util import safe_request
# -*- coding: utf-8 -*-
# Copyright 2014, 2015 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images and ugoira from http://www.pixiv.net/"""
from .common import AsynchronousExtractor
from .common import Message
from .common import safe_request
import re
import csv
import requests
class Extractor(AsyncExtractor):
info = {
"category": "pixiv",
"extractor": "PixivExtractor",
"directory": ["{category}", "{artist-id}"],
"filename": "{category}_{artist-id}_{illust-id}{num}.{extension}",
"pattern": [
r"(?:https?://)?(?:www\.)?pixiv\.net/member(?:_illust)?\.php\?id=(\d+)",
],
}
class PixivExtractor(AsynchronousExtractor):
member_url = "http://www.pixiv.net/member_illust.php"
illust_url = "http://www.pixiv.net/member_illust.php?mode=medium"
def __init__(self, match, config):
AsyncExtractor.__init__(self, config)
self.member_id = match.group(1)
self.category = "pixiv"
self.directory = self.member_id
self.session.cookies.update(config["pixiv-cookies"])
self.session.headers.update({"Referer": "http://www.pixiv.net/"})
self.api = PixivAPI(config["pixiv-cookies"]["PHPSESSID"])
singl_v1_fmt = ("http://i{thumbnail-url[8]}.pixiv.net/img{directory:>02}"
"/img/{artist-nick}/{illust-id}.{extension}")
manga_v1_fmt = ("http://i{thumbnail-url[8]}.pixiv.net/img{directory:>02}"
"/img/{artist-nick}/{illust-id}{big}_p{index}.{extension}")
def images(self):
sname_fmt = "pixiv_{1}_{0}.{2}"
mname_fmt = "pixiv_{1}_{0}_p{num:02}.{2}"
singl_v2_fmt = ("http://i{thumbnail-url[8]}.pixiv.net/img-original/img"
"/{url-date}/{illust-id}_p0.{extension}")
manga_v2_fmt = ("http://i{thumbnail-url[8]}.pixiv.net/img-original/img"
"/{url-date}/{illust-id}_p{index}.{extension}")
singl_v1_fmt = "http://i{6[8]}.pixiv.net/img{4:>02}/img/{24}/{0}.{2}"
manga_v1_fmt = "http://i{6[8]}.pixiv.net/img{4:>02}/img/{24}/{0}{big}_p{num}.{2}"
singl_v2_fmt = "http://i{6[8]}.pixiv.net/img-original/img/{date}/{0}_p0.{2}"
manga_v2_fmt = "http://i{6[8]}.pixiv.net/img-original/img/{date}/{0}_p{num}.{2}"
def __init__(self, match, config):
AsynchronousExtractor.__init__(self, config)
self.config = config
self.artist_id = match.group(1)
self.api = PixivAPI(config["pixiv-cookies"]["PHPSESSID"])
date = ""
big = ""
def items(self):
yield Message.Version, 1
yield Message.Headers, {"Referer": "http://www.pixiv.net/"}
yield Message.Cookies, self.config["pixiv-cookies"]
yield Message.Directory, self.get_job_metadata()
for img in self.image_ids():
data = self.api.request(img)
for illust_id in self.get_illust_ids():
data = self.api.request(illust_id)
# debug
# for i, value in enumerate(data):
# print("{:02}: {}".format(i, value))
# return
# debug end
if "うごイラ" in data[13]:
# if "うごイラ" in data["tags"]:
# ugoira / animations
try:
url, framelist = self.parse_ugoira(img)
data[2] = "zip"
yield (url, sname_fmt.format(*data))
data[2] = "txt"
yield (framelist, sname_fmt.format(*data))
continue
except:
print("[Warning] failed to get ugoira url; trying fallback")
# url, framelist = self.parse_ugoira(img)
# data[2] = "zip"
# yield (url, sname_fmt.format(*data))
# data[2] = "txt"
# yield (framelist, sname_fmt.format(*data))
# continue
# images
if img > 46270949:
date = data[6][45:64]
url_s_fmt = singl_v2_fmt
url_m_fmt = manga_v2_fmt
if illust_id > 46270949:
big = ""
url_s_fmt = self.singl_v2_fmt
url_m_fmt = self.manga_v2_fmt
else:
big = "_big" if img > 11319935 else ""
url_s_fmt = singl_v1_fmt
url_m_fmt = manga_v1_fmt
big = "_big" if illust_id > 11319935 else ""
url_s_fmt = self.singl_v1_fmt
url_m_fmt = self.manga_v1_fmt
if not data[19]:
yield (url_s_fmt.format(*data, date=date), sname_fmt.format(*data))
if not data["count"]:
yield Message.Url, url_s_fmt.format(**data), data
else:
for i in range(0, int(data[19])):
yield (url_m_fmt.format(*data, num=i, date=date, big=big),
mname_fmt.format(*data, num=i))
def image_ids(self):
"""generator -- yield all image ids"""
needle = '<a href="/member_illust.php?mode=medium&amp;illust_id='
params = {"id": self.member_id, "p": 1}
for i in range(0, int(data["count"])):
data["num"] = "_p{:02}".format(i)
yield (Message.Url,
url_m_fmt.format(index=i, big=big, **data),
data.copy())
def get_illust_ids(self):
"""Yield all illust-ids for a pixiv-member"""
needle = ('<li class="image-item "><a href="'
'/member_illust.php?mode=medium&amp;illust_id=')
params = {"id": self.artist_id, "p": 1}
pos = 0
while True:
text = self.request(self.member_url, params=params).text
end = 0
text = self.request(self.member_url, params=params).text
found = 0
while True:
pos = text.find(needle, end)
if pos == -1:
illust_id, pos = self.extract(text, needle, '"', pos)
if illust_id is None:
break
pos += len(needle)
end = text.find('"', pos)
found += 1
yield int(text[pos:end])
yield int(illust_id)
if found != 20:
return
params["p"] += 1
def parse_ugoira(self, illust_id):
"""Parse ugoira data"""
# get illust page
text = self.request(
self.illust_url,
@ -108,9 +130,15 @@ class Extractor(AsyncExtractor):
r'\1 \2\n',
frames
)
return url, framelist
def get_job_metadata(self):
"""Collect metadata for extractor-job"""
return {
"category": info["category"],
"artist-id": self.artist_id,
}
class PixivAPI():
api_url = "http://spapi.pixiv.net/iphone/illust.php"
@ -120,31 +148,34 @@ class PixivAPI():
self.session.params["PHPSESSID"] = session_id
def request(self, illust_id):
while True:
data = next(csv.reader(
[self.api_call(illust_id)]
))
return {
"category": info["category"],
"illust-id": data[0],
"artist-id": data[1],
"extension": data[2],
"title": data[3],
"directory": data[4],
"artist-name": data[5],
"thumbnail-url": data[6],
"url-date": data[6][45:64],
# "thumbnail-mobile-url": data[9],
"date": data[12],
"tags": data[13],
# "description": data[18],
"count": data[19],
"artist-nick": data[24],
# "artist-avatar-url": data[29],
"num": "",
}
def api_call(self, illust_id):
text = ""
while len(text) < 32:
text = safe_request(
self.session,
self.api_url,
self.session, self.api_url,
params={"illust_id": illust_id}
).text
if len(text) > 31:
return next(csv.reader([text]))
# class FileDict(dict):
#
# def __init__(self, *args):
# super().__init__()
# self.re = re.compile(r"pixiv_\d+_(?P<id>\d+)(?P<extra>_p\d+)?\.[a-z]{3}")
# for arg in args:
# self.load_from(arg)
#
# def load_from(self, directory):
# match = self.re.match
# for file in os.listdir(directory):
# m = match(file)
# if m is None:
# continue
# val = True if m.group("extra") else False
# dict.__setitem__(self, m.group("id"), val)
#
# def __getitem__(self, key):
# return dict.get(self, key)
return text

Write Preview
Loading…
Cancel
Save