From 138135c190bf05d9b5429945a777498a5313aa11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 11 Feb 2020 19:51:24 +0100 Subject: [PATCH] [furaffinity] add extractors (#284) --- docs/supportedsites.rst | 1 + gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/furaffinity.py | 180 ++++++++++++++++++++++++++++ scripts/supportedsites.py | 1 + 4 files changed, 183 insertions(+) create mode 100644 gallery_dl/extractor/furaffinity.py diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 59bcc330..13fa9cc7 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -33,6 +33,7 @@ Fallen Angels Scans https://www.fascans.com/ Chapters, Manga Fashion Nova https://www.fashionnova.com/ Collections, Products Fireden https://boards.fireden.net/ Threads Flickr https://www.flickr.com/ |flickr-C| Optional (OAuth) +Fur Affinity https://www.furaffinity.net/ Favorites, Galleries, Posts, Scraps, User Profiles Fuskator https://fuskator.com/ Galleries, Search Results Futaba Channel https://www.2chan.net/ Threads Gelbooru https://gelbooru.com/ Pools, Posts, Tag-Searches diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 790207d9..086d0d0f 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -30,6 +30,7 @@ modules = [ "exhentai", "fallenangels", "flickr", + "furaffinity", "fuskator", "gelbooru", "gfycat", diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py new file mode 100644 index 00000000..8a1e6b32 --- /dev/null +++ b/gallery_dl/extractor/furaffinity.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- + +# Copyright 2020 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.furaffinity.net/""" + +from .common import Extractor, Message +from .. import text, util + + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?furaffinity\.net" + + +class FuraffinityExtractor(Extractor): + """Base class for furaffinity extractors""" + category = "furaffinity" + directory_fmt = ("{category}", "{user!l}") + filename_fmt = "{id} {title}.{extension}" + archive_fmt = "{id}" + root = "https://www.furaffinity.net" + + def __init__(self, match): + Extractor.__init__(self, match) + self.user = match.group(1) + self.offset = 0 + + def items(self): + for post_id in util.advance(self.posts(), self.offset): + post = self._parse_post(post_id) + if post: + yield Message.Directory, post + text.nameext_from_url(post["url"], post) + yield Message.Url, post["url"], post + + def posts(self): + return self._pagination() + + def skip(self, num): + self.offset += num + return num + + def _parse_post(self, post_id): + url = "{}/view/{}/".format(self.root, post_id) + extr = text.extract_from(self.request(url).text) + + title, _, artist = text.unescape(extr( + 'property="og:title" content="', '"')).rpartition(" by ") + if not extr('class="download', '>'): + self.log.warning( + "Unable to download post %s (\"%s\")", post_id, + text.remove_html(extr('class="link-override">', '

'))) + return None + + return { + "id" : text.parse_int(post_id), + "title" : title, + "artist": artist, + "user" : self.user or artist, + "url" : "https:" + extr('href="', '"'), + "tags" : text.split_html(extr('class="tags-row">', '')), + "date" : text.parse_datetime(extr( + '', ''), "", "")), + } + + def _pagination(self): + num = 1 + + while True: + url = "{}/{}/{}/{}/".format( + self.root, self.subcategory, self.user, num) + page = self.request(url).text + post_id = None + + for post_id in text.extract_iter(page, 'id="sid-', '"'): + yield post_id + + if not post_id: + return + num += 1 + + def _pagination_favorites(self): + path = "/favorites/{}/".format(self.user) + + while path: + page = self.request(self.root + path).text + yield from text.extract_iter(page, 'id="sid-', '"') + path = text.extract(page, 'button standard right" href="', '"')[0] + + +class FuraffinityGalleryExtractor(FuraffinityExtractor): + """Extractor for a furaffinity user's gallery""" + subcategory = "gallery" + pattern = BASE_PATTERN + r"/gallery/([^/?&#]+)" + test = ("https://www.furaffinity.net/gallery/mirlinthloth/", { + "pattern": r"https://d.facdn.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "range": "45-50", + "count": 6, + }) + + +class FuraffinityScrapsExtractor(FuraffinityExtractor): + """Extractor for a furaffinity user's scraps""" + subcategory = "scraps" + directory_fmt = ("{category}", "{user!l}", "Scraps") + pattern = BASE_PATTERN + r"/scraps/([^/?&#]+)" + test = ("https://www.furaffinity.net/scraps/mirlinthloth/", { + "pattern": r"https://d.facdn.net/art/[^/]+(/stories)?/\d+/\d+.\w+.\w+", + "count": ">= 3", + }) + + +class FuraffinityFavoriteExtractor(FuraffinityExtractor): + """Extractor for a furaffinity user's favorites""" + subcategory = "favorite" + directory_fmt = ("{category}", "{user!l}", "Favorites") + pattern = BASE_PATTERN + r"/favorites/([^/?&#]+)" + test = ("https://www.furaffinity.net/favorites/mirlinthloth/", { + "pattern": r"https://d.facdn.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "range": "45-50", + "count": 6, + }) + + def posts(self): + return self._pagination_favorites() + + +class FuraffinityPostExtractor(FuraffinityExtractor): + """Extractor for individual posts on furaffinity""" + subcategory = "post" + pattern = BASE_PATTERN + r"/view/(\d+)" + test = ("https://www.furaffinity.net/view/21835115/", { + "url": "eae4ef93d99365c69b31a37561bd800c03d336ad", + "keyword": { + "artist" : "mirlinthloth", + "date" : "type:datetime", + "description": "A Song made playing the game Cosmic DJ.", + "extension" : "mp3", + "filename" : r"re:\d+\.mirlinthloth_dj_fennmink_-_bude_s_4_ever", + "id" : 21835115, + "tags" : list, + "title" : "Bude's 4 Ever", + "url" : "re:https://d.facdn.net/art/mirlinthloth/music/", + "user" : "mirlinthloth", + }, + }) + + def posts(self): + post_id = self.user + self.user = None + return (post_id,) + + +class FuraffinityUserExtractor(FuraffinityExtractor): + """Extractor for furaffinity user profiles""" + subcategory = "user" + pattern = BASE_PATTERN + r"/user/([^/?&#]+)" + test = ( + ("https://www.furaffinity.net/user/mirlinthloth/", { + "pattern": r"/gallery/mirlinthloth/$", + }), + ("https://www.furaffinity.net/user/mirlinthloth/", { + "options": (("include", "all"),), + "pattern": r"/(gallery|scraps|favorites)/mirlinthloth/$", + "count": 3, + }), + ) + + def items(self): + base = "{}/{{}}/{}/".format(self.root, self.user) + return self._dispatch_extractors(( + (FuraffinityGalleryExtractor , base.format("gallery")), + (FuraffinityScrapsExtractor , base.format("scraps")), + (FuraffinityFavoriteExtractor, base.format("favorites")), + ), ("gallery",)) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 247717b3..a1862c9c 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -28,6 +28,7 @@ CATEGORY_MAP = { "exhentai" : "ExHentai", "fallenangels" : "Fallen Angels Scans", "fashionnova" : "Fashion Nova", + "furaffinity" : "Fur Affinity", "hbrowse" : "HBrowse", "hentai2read" : "Hentai2Read", "hentaicafe" : "Hentai Cafe",