# -*- coding: utf-8 -*- # Copyright 2014-2017 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extract images from https://chan.sankakucomplex.com/""" from .common import AsynchronousExtractor, Message from .. import text class SankakuTagExtractor(AsynchronousExtractor): """Extractor for images from chan.sankakucomplex.com by search-tags""" category = "sankaku" subcategory = "tag" directory_fmt = ["{category}", "{tags}"] filename_fmt = "{category}_{id}_{md5}.{extension}" pattern = [r"(?:https?://)?chan\.sankakucomplex\.com/\?tags=([^&]+)"] test = [("https://chan.sankakucomplex.com/?tags=bonocho", { "url": "2561ca0d8482ead48f22a7abcd23919cd78344a1", "keyword": "5e3a39fdc6698e63ed0054478ebd4ca632ce643e", })] url = "https://chan.sankakucomplex.com/" def __init__(self, match): AsynchronousExtractor.__init__(self) self.tags = text.unquote(match.group(1)) self.session.headers["User-Agent"] = ( "Mozilla/5.0 Gecko/20100101 Firefox/40.0" ) def items(self): data = self.get_job_metadata() yield Message.Version, 1 yield Message.Directory, data for image in self.get_images(): image.update(data) yield Message.Url, image["file-url"], image def get_job_metadata(self): """Collect metadata for extractor-job""" return {"tags": self.tags} def get_images(self): params = { "tags": self.tags, "page": 1, } while True: count = 0 page = self.request(self.url, params=params).text pos = text.extract(page, '