# -*- coding: utf-8 -*- # Copyright 2015 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extract images from http://www.deviantart.com/""" from .common import Extractor, AsynchronousExtractor, Message from .. import text import re class DeviantArtUserExtractor(AsynchronousExtractor): """Extract all works of an artist on deviantart""" category = "deviantart" subcategory = "user" directory_fmt = ["{category}", "{artist}"] filename_fmt = "{category}_{index}_{title}.{extension}" pattern = [r"(?:https?://)?([^\.]+)\.deviantart\.com(?:/gallery)?/?$"] def __init__(self, match): AsynchronousExtractor.__init__(self) self.session.cookies["agegate_state"] = "1" self.artist = match.group(1) def items(self): metadata = self.get_job_metadata() yield Message.Version, 1 yield Message.Directory, metadata for url, data in self.get_works(): data.update(metadata) yield Message.Url, url, data def get_works(self): """Yield all work-items for a deviantart-artist""" url = "http://{}.deviantart.com/gallery/".format(self.artist) params = {"catpath": "/", "offset": 0} while True: page = self.request(url, params=params).text _, pos = text.extract(page, '
', ''), ('date' , 'title="', '"'), ), values={'category': self.category, "index": self.index})[0] data["description"] = text.unescape(text.unescape(data["description"])) data["artist"] = text.extract(data["url"], "//", ".")[0] data["date"] = text.extract(data["date"], ", ", " in ", len(data["title"]))[0] text.nameext_from_url(data["image"], data) yield Message.Version, 1 yield Message.Directory, data yield Message.Url, data["image"], data