# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for nijie instances""" from .common import BaseExtractor, Message, AsynchronousMixin from .. import text, exception from ..cache import cache class NijieExtractor(AsynchronousMixin, BaseExtractor): """Base class for nijie extractors""" basecategory = "Nijie" directory_fmt = ("{category}", "{user_id}") filename_fmt = "{image_id}_p{num}.{extension}" archive_fmt = "{image_id}_{num}" request_interval = (1.0, 2.0) def __init__(self, match): BaseExtractor.__init__(self, match) self.user_id = text.parse_int(match.group(match.lastindex)) def initialize(self): self.cookies_domain = "." + self.root.rpartition("/")[2] self.cookies_names = (self.category + "_tok",) BaseExtractor.initialize(self) self.user_name = None if self.category == "horne": self._extract_data = self._extract_data_horne def items(self): self.login() for image_id in self.image_ids(): url = "{}/view.php?id={}".format(self.root, image_id) response = self.request(url, fatal=False) if response.status_code >= 400: continue page = response.text data = self._extract_data(page) data["image_id"] = text.parse_int(image_id) if self.user_name: data["user_id"] = self.user_id data["user_name"] = self.user_name else: data["user_id"] = data["artist_id"] data["user_name"] = data["artist_name"] yield Message.Directory, data for image in self._extract_images(page): image.update(data) if not image["extension"]: image["extension"] = "jpg" yield Message.Url, image["url"], image def image_ids(self): """Collect all relevant image-ids""" @staticmethod def _extract_data(page): """Extract image metadata from 'page'""" extr = text.extract_from(page) keywords = text.unescape(extr( 'name="keywords" content="', '" />')).split(",") data = { "title" : keywords[0].strip(), "description": text.unescape(extr( '"description": "', '"').replace("&", "&")), "date" : text.parse_datetime(extr( '"datePublished": "', '"'), "%a %b %d %H:%M:%S %Y", 9), "artist_id" : text.parse_int(extr('/members.php?id=', '"')), "artist_name": keywords[1], "tags" : keywords[2:-1], } return data @staticmethod def _extract_data_horne(page): """Extract image metadata from 'page'""" extr = text.extract_from(page) keywords = text.unescape(extr( 'name="keywords" content="', '" />')).split(",") data = { "title" : keywords[0].strip(), "description": text.unescape(extr( 'property="og:description" content="', '"')), "artist_id" : text.parse_int(extr('members.php?id=', '"')), "artist_name": keywords[1], "tags" : keywords[2:-1], "date" : text.parse_datetime(extr( "itemprop='datePublished' content=", "<").rpartition(">")[2], "%Y-%m-%d %H:%M:%S", 9), } return data @staticmethod def _extract_images(page): """Extract image URLs from 'page'""" images = text.extract_iter(page, "/view_popup.php", "") for num, image in enumerate(images): src = text.extr(image, 'src="', '"') if not src: continue url = ("https:" + src).replace("/__rs_l120x120/", "/") yield text.nameext_from_url(url, { "num": num, "url": url, }) @staticmethod def _extract_user_name(page): return text.unescape(text.extr(page, "
", "<")) def login(self): if self.cookies_check(self.cookies_names): return username, password = self._get_auth_info() self.cookies_update(self._login_impl(username, password)) @cache(maxage=90*24*3600, keyarg=1) def _login_impl(self, username, password): if not username or not password: raise exception.AuthenticationError( "Username and password required") self.log.info("Logging in as %s", username) url = "{}/login_int.php".format(self.root) data = {"email": username, "password": password, "save": "on"} response = self.request(url, method="POST", data=data) if "/login.php" in response.text: raise exception.AuthenticationError() return self.cookies def _pagination(self, path): url = "{}/{}.php".format(self.root, path) params = {"id": self.user_id, "p": 1} while True: page = self.request(url, params=params, notfound="artist").text if self.user_name is None: self.user_name = self._extract_user_name(page) yield from text.extract_iter(page, 'illust_id="', '"') if '