# -*- coding: utf-8 -*- # Copyright 2015-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extract images from https://nijie.info/""" from .common import Extractor, Message, AsynchronousMixin from .. import text, exception from ..cache import cache BASE_PATTERN = r"(?:https?://)?(?:www\.)?nijie\.info" class NijieExtractor(AsynchronousMixin, Extractor): """Base class for nijie extractors""" category = "nijie" directory_fmt = ("{category}", "{user_id}") filename_fmt = "{image_id}_p{num}.{extension}" archive_fmt = "{image_id}_{num}" cookiedomain = "nijie.info" cookienames = ("nemail", "nlogin") root = "https://nijie.info" view_url = "https://nijie.info/view.php?id=" popup_url = "https://nijie.info/view_popup.php?id=" def __init__(self, match): Extractor.__init__(self, match) self.user_id = text.parse_int(match.group(1)) self.user_name = None self.session.headers["Referer"] = self.root + "/" def items(self): self.login() yield Message.Version, 1 for image_id in self.image_ids(): response = self.request(self.view_url + image_id, fatal=False) if response.status_code >= 400: continue page = response.text data = self._extract_data(page) data["image_id"] = text.parse_int(image_id) yield Message.Directory, data for image in self._extract_images(page): image.update(data) if not image["extension"]: image["extension"] = "jpg" yield Message.Url, image["url"], image def image_ids(self): """Collect all relevant image-ids""" @staticmethod def _extract_data(page): """Extract image metadata from 'page'""" extr = text.extract_from(page) keywords = text.unescape(extr( 'name="keywords" content="', '" />')).split(",") data = { "title" : keywords[0].strip(), "description": text.unescape(extr( '"description": "', '"').replace("&", "&")), "date" : text.parse_datetime(extr( '"datePublished": "', '"') + "+0900", "%a %b %d %H:%M:%S %Y%z"), "artist_id" : text.parse_int(extr( '"sameAs": "https://nijie.info/members.php?id=', '"')), "artist_name": keywords[1], "tags" : keywords[2:-1], } data["user_id"] = data["artist_id"] data["user_name"] = data["artist_name"] return data @staticmethod def _extract_images(page): """Extract image URLs from 'page'""" images = text.extract_iter(page, '', '<')[0] or "") yield from text.extract_iter(page, 'illust_id="', '"') if '