# -*- coding: utf-8 -*- # Copyright 2015-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://nijie.info/""" from .common import Extractor, Message, AsynchronousMixin from .. import text, exception from ..cache import cache BASE_PATTERN = r"(?:https?://)?(?:www\.)?nijie\.info" class NijieExtractor(AsynchronousMixin, Extractor): """Base class for nijie extractors""" category = "nijie" directory_fmt = ("{category}", "{user_id}") filename_fmt = "{image_id}_p{num}.{extension}" archive_fmt = "{image_id}_{num}" cookiedomain = "nijie.info" cookienames = ("nemail", "nlogin") root = "https://nijie.info" view_url = "https://nijie.info/view.php?id=" popup_url = "https://nijie.info/view_popup.php?id=" def __init__(self, match): Extractor.__init__(self, match) self.user_id = text.parse_int(match.group(1)) self.user_name = None self.session.headers["Referer"] = self.root + "/" def items(self): self.login() for image_id in self.image_ids(): response = self.request(self.view_url + image_id, fatal=False) if response.status_code >= 400: continue page = response.text data = self._extract_data(page) data["image_id"] = text.parse_int(image_id) yield Message.Directory, data for image in self._extract_images(page): image.update(data) if not image["extension"]: image["extension"] = "jpg" yield Message.Url, image["url"], image def image_ids(self): """Collect all relevant image-ids""" @staticmethod def _extract_data(page): """Extract image metadata from 'page'""" extr = text.extract_from(page) keywords = text.unescape(extr( 'name="keywords" content="', '" />')).split(",") data = { "title" : keywords[0].strip(), "description": text.unescape(extr( '"description": "', '"').replace("&", "&")), "date" : text.parse_datetime(extr( '"datePublished": "', '"') + "+0900", "%a %b %d %H:%M:%S %Y%z"), "artist_id" : text.parse_int(extr( '"sameAs": "https://nijie.info/members.php?id=', '"')), "artist_name": keywords[1], "tags" : keywords[2:-1], } data["user_id"] = data["artist_id"] data["user_name"] = data["artist_name"] return data @staticmethod def _extract_images(page): """Extract image URLs from 'page'""" images = text.extract_iter(page, '", "<")[0] or "") def login(self): """Login and obtain session cookies""" if not self._check_cookies(self.cookienames): username, password = self._get_auth_info() self._update_cookies(self._login_impl(username, password)) @cache(maxage=150*24*3600, keyarg=1) def _login_impl(self, username, password): if not username or not password: raise exception.AuthenticationError( "Username and password required") self.log.info("Logging in as %s", username) url = "{}/login_int.php".format(self.root) data = {"email": username, "password": password, "save": "on"} response = self.request(url, method="POST", data=data) if "//nijie.info/login.php" in response.text: raise exception.AuthenticationError() return self.session.cookies def _pagination(self, path): url = "{}/{}.php".format(self.root, path) params = {"id": self.user_id, "p": 1} while True: page = self.request(url, params=params, notfound="artist").text if self.user_name is None: self.user_name = self._extract_user_name(page) yield from text.extract_iter(page, 'illust_id="', '"') if '