# -*- coding: utf-8 -*- # Copyright 2018 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.newgrounds.com/""" from .common import Extractor, Message from .. import text import json class NewgroundsExtractor(Extractor): """Base class for newgrounds extractors""" category = "newgrounds" directory_fmt = ["{category}", "{user}"] filename_fmt = "{category}_{index}_{title}.{extension}" archive_fmt = "{index}" def __init__(self, match): Extractor.__init__(self) self.user = match.group(1) self.root = "https://{}.newgrounds.com".format(self.user) def items(self): data = self.get_metadata() yield Message.Version, 1 yield Message.Directory, data for page_url in self.get_page_urls(): url, image = self.parse_page_data(page_url) image.update(data) yield Message.Url, url, text.nameext_from_url(url, image) def get_metadata(self): """Collect metadata for extractor-job""" return {"user": self.user} def get_page_urls(self): """Return urls of all relevant image pages""" def parse_page_data(self, page_url): """Collect url and metadata from an image page""" def _pagination(self, url): headers = { "Referer": self.root, "X-Requested-With": "XMLHttpRequest", "Accept": "application/json, text/javascript, */*; q=0.01", } while True: data = self.request(url, headers=headers).json() for year in data["sequence"]: for item in data["years"][str(year)]["items"]: page_url = text.extract(item, 'href="', '"')[0] yield text.urljoin(self.root, page_url) if not data["more"]: return url = text.urljoin(self.root, data["more"]) class NewgroundsUserExtractor(NewgroundsExtractor): """Extractor for all images of a newgrounds user""" subcategory = "user" pattern = [r"(?:https?://)?([^.]+)\.newgrounds\.com(?:/art)?/?$"] test = [ ("https://blitzwuff.newgrounds.com/art", { "url": "24b19c4a135a09889fac7b46a74e427e4308d02b", "keyword": "7cbadd5426b8cba270995fffca2e52571d1f9c8d", }), ("https://derpixon.newgrounds.com/art", None), ("https://derpixon.newgrounds.com/", None), ] def get_page_urls(self): return self._pagination(self.root + "/art/page/1") def parse_page_data(self, page_url): page = self.request(page_url).text full, pos = text.extract(page, '"full_image_text":', '});') desc, pos = text.extract(page, '"og:description" content="', '"', pos) rate, pos = text.extract(page, 'class="rated-', '"', pos) tags, pos = text.extract(page, '