# -*- coding: utf-8 -*- # Copyright 2014-2017 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extract images from https://chan.sankakucomplex.com/""" from .common import Extractor, Message from .. import text, util, exception from ..cache import cache import time import random class SankakuTagExtractor(Extractor): """Extractor for images from chan.sankakucomplex.com by search-tags""" category = "sankaku" subcategory = "tag" directory_fmt = ["{category}", "{tags}"] filename_fmt = "{category}_{id}_{md5}.{extension}" pattern = [r"(?:https?://)?chan\.sankakucomplex\.com" r"/\?(?:[^&#]*&)*tags=([^&#]+)"] test = [("https://chan.sankakucomplex.com/?tags=bonocho", { "count": 5, "pattern": (r"https://cs\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+"), })] root = "https://chan.sankakucomplex.com" cookienames = ("login", "pass_hash") cookiedomain = "chan.sankakucomplex.com" def __init__(self, match): Extractor.__init__(self) self.logged_in = True self.pagestart = 1 self.tags = text.unquote(match.group(1).replace("+", " ")) self.wait_min = self.config("wait-min", 2) self.wait_max = self.config("wait-max", 4) if self.wait_max < self.wait_min: self.wait_max = self.wait_min def skip(self, num): pages = min(num // 20, 49) self.pagestart += pages return pages * 20 def items(self): self.login() data = self.get_job_metadata() yield Message.Version, 1 yield Message.Directory, data for image in self.get_images(): image.update(data) yield Message.Url, image["file_url"], image def get_job_metadata(self): """Collect metadata for extractor-job""" return {"tags": self.tags} def get_images(self): """Yield all available images for the given tags""" params = { "tags": self.tags, "page": self.pagestart, } while self.logged_in or params["page"] <= 25: image = None page = self.request(self.root, params=params, retries=10).text pos = text.extract(page, '