From 92f98e6f5e76ab503b8e8531812c26271c07e0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 21 Aug 2023 23:46:39 +0200 Subject: [PATCH 01/53] 'sys.exit' -> 'SystemExit' --- gallery_dl/__init__.py | 2 +- gallery_dl/__main__.py | 4 ++-- gallery_dl/actions.py | 3 +-- gallery_dl/config.py | 4 ++-- gallery_dl/job.py | 2 +- gallery_dl/postprocessor/compare.py | 5 ++--- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index ab72b8e1..62622b9a 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -297,7 +297,7 @@ def main(): return retval except KeyboardInterrupt: - sys.exit("\nKeyboardInterrupt") + raise SystemExit("\nKeyboardInterrupt") except BrokenPipeError: pass except OSError as exc: diff --git a/gallery_dl/__main__.py b/gallery_dl/__main__.py index 637d463f..441009fe 100644 --- a/gallery_dl/__main__.py +++ b/gallery_dl/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -17,4 +17,4 @@ if __package__ is None and not hasattr(sys, "frozen"): import gallery_dl if __name__ == "__main__": - sys.exit(gallery_dl.main()) + raise SystemExit(gallery_dl.main()) diff --git a/gallery_dl/actions.py b/gallery_dl/actions.py index 15ca31ec..883e38b0 100644 --- a/gallery_dl/actions.py +++ b/gallery_dl/actions.py @@ -9,7 +9,6 @@ """ """ import re -import sys import logging import operator from . import util, exception @@ -98,7 +97,7 @@ def action_exit(opts): pass def _exit(args): - sys.exit(opts) + raise SystemExit(opts) return _exit diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 767185fb..4be6c532 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -100,12 +100,12 @@ def load(files=None, strict=False, loads=util.json_loads): except OSError as exc: if strict: log.error(exc) - sys.exit(1) + raise SystemExit(1) except Exception as exc: log.error("%s when loading '%s': %s", exc.__class__.__name__, path, exc) if strict: - sys.exit(2) + raise SystemExit(2) else: if not _config: _config.update(conf) diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 459db5c7..af924151 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -515,7 +515,7 @@ class DownloadJob(Job): elif skip == "terminate": self._skipexc = exception.TerminateExtraction elif skip == "exit": - self._skipexc = sys.exit + self._skipexc = SystemExit self._skipmax = text.parse_int(smax) else: # monkey-patch methods to always return False diff --git a/gallery_dl/postprocessor/compare.py b/gallery_dl/postprocessor/compare.py index 910e1d77..3bb63c80 100644 --- a/gallery_dl/postprocessor/compare.py +++ b/gallery_dl/postprocessor/compare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2021 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,7 +10,6 @@ from .common import PostProcessor from .. import text, util, exception -import sys import os @@ -31,7 +30,7 @@ class ComparePP(PostProcessor): elif equal == "terminate": self._equal_exc = exception.TerminateExtraction elif equal == "exit": - self._equal_exc = sys.exit + self._equal_exc = SystemExit job.register_hooks({"file": ( self.enumerate From fdfb22c91fe06abcc3488485f3c32d937803b50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 23 Aug 2023 12:29:32 +0200 Subject: [PATCH 02/53] [instagram] fix video preview archive IDs (#2135, #4455) --- gallery_dl/extractor/instagram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index a00d73f8..db9333fd 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -90,7 +90,9 @@ class InstagramExtractor(Extractor): file["_http_headers"] = video_headers text.nameext_from_url(url, file) yield Message.Url, url, file - if not previews: + if previews: + file["media_id"] += "p" + else: continue url = file["display_url"] From fd6b413f3c70ce1a365f5a22aec155cd30f05375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 23 Aug 2023 12:33:29 +0200 Subject: [PATCH 03/53] [exhentai] fix 'domain' option (#4458) regression from a383eca7 --- gallery_dl/extractor/exhentai.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index d5f1d02b..5ee008d1 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -482,7 +482,6 @@ class ExhentaiSearchExtractor(ExhentaiExtractor): def __init__(self, match): ExhentaiExtractor.__init__(self, match) - self.search_url = self.root _, query, tag = match.groups() if tag: @@ -497,6 +496,9 @@ class ExhentaiSearchExtractor(ExhentaiExtractor): if "next" not in self.params: self.params["page"] = text.parse_int(self.params.get("page")) + def _init(self): + self.search_url = self.root + def items(self): self.login() data = {"_extractor": ExhentaiGalleryExtractor} @@ -542,6 +544,5 @@ class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor): "&f_apply=Search+Favorites"), ) - def __init__(self, match): - ExhentaiSearchExtractor.__init__(self, match) + def _init(self): self.search_url = self.root + "/favorites.php" From 5ed245317df5e3edb644d0aab4bb35a759d5eaee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 23 Aug 2023 12:54:45 +0200 Subject: [PATCH 04/53] [exhentai] add 'fav' option (#4409) The name 'favorite' is already taken as extractor subcategory --- docs/configuration.rst | 16 ++++++++++++++++ gallery_dl/extractor/exhentai.py | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c7f3712..5f8ac6eb 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1475,6 +1475,22 @@ Description * ``"exhentai.org"``: Use ``exhentai.org`` for all URLs +extractor.exhentai.fav +---------------------- +Type + ``string`` +Example + ``"4"`` +Description + After downloading a gallery, + add it to your account's favorites as the given category number. + + Note: Set this to `"favdel"` to remove galleries from your favorites. + + Note: This will remove any Favorite Notes when applied + to already favorited galleries. + + extractor.exhentai.limits ------------------------- Type diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 5ee008d1..0a60ed82 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -179,6 +179,20 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): if source == "hitomi": self.items = self._items_hitomi + def favorite(self, slot="0"): + url = self.root + "/gallerypopups.php" + params = { + "gid": self.gallery_id, + "t" : self.gallery_token, + "act": "addfav", + } + data = { + "favcat" : slot, + "apply" : "Apply Changes", + "update" : "1", + } + self.request(url, method="POST", params=params, data=data) + def items(self): self.login() @@ -223,6 +237,10 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): data["_http_validate"] = None yield Message.Url, url, data + fav = self.config("fav") + if fav is not None: + self.favorite(fav) + def _items_hitomi(self): if self.config("metadata", False): data = self.metadata_from_api() From e0829ff0fd4dfa793e9e9d4b0addb8c5f707791f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 23 Aug 2023 16:16:30 +0200 Subject: [PATCH 05/53] [twitter] add 'date_original' metadata for retweets (#4337, #4443) --- gallery_dl/extractor/twitter.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 81b5291b..34ec4162 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -336,6 +336,9 @@ class TwitterExtractor(Extractor): tdata["reply_to"] = tweet["in_reply_to_screen_name"] if "quoted_by" in tweet: tdata["quote_by"] = tweet["quoted_by"] + if tdata["retweet_id"]: + tdata["date_original"] = text.parse_timestamp( + ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) return tdata @@ -886,9 +889,10 @@ class TwitterTweetExtractor(TwitterExtractor): "options": (("retweets", True),), "count": 2, "keyword": { - "tweet_id" : 1296304589591810048, - "retweet_id": 1296296016002547713, - "date" : "dt:2020-08-20 04:34:32", + "tweet_id" : 1296304589591810048, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:34:32", + "date_original": "dt:2020-08-20 04:00:28", }, }), # original retweets (#1026) @@ -896,9 +900,10 @@ class TwitterTweetExtractor(TwitterExtractor): "options": (("retweets", "original"),), "count": 2, "keyword": { - "tweet_id" : 1296296016002547713, - "retweet_id": 1296296016002547713, - "date" : "dt:2020-08-20 04:00:28", + "tweet_id" : 1296296016002547713, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:00:28", + "date_original": "dt:2020-08-20 04:00:28", }, }), # all Tweets from a 'conversation' (#1319) From 3a271504791f99786e980d3677fca4a79ebea524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 23 Aug 2023 23:55:39 +0200 Subject: [PATCH 06/53] [instagram] add 'following' extractor (#1848) --- gallery_dl/extractor/instagram.py | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index db9333fd..d9049678 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -594,6 +594,23 @@ class InstagramHighlightsExtractor(InstagramExtractor): return self.api.highlights_media(uid) +class InstagramFollowingExtractor(InstagramExtractor): + """Extractor for an Instagram user's followed users""" + subcategory = "following" + pattern = USER_PATTERN + r"/following" + test = ("https://www.instagram.com/instagram/following", { + "range": "1-16", + "count": ">= 16", + }) + + def items(self): + uid = self.api.user_id(self.item) + for user in self.api.user_following(uid): + user["_extractor"] = InstagramUserExtractor + url = "{}/{}".format(self.root, user["username"]) + yield Message.Queue, url, user + + class InstagramTagExtractor(InstagramExtractor): """Extractor for Instagram tags""" subcategory = "tag" @@ -871,6 +888,11 @@ class InstagramRestAPI(): params = {"count": 30} return self._pagination(endpoint, params) + def user_following(self, user_id): + endpoint = "/v1/friendships/{}/following/".format(user_id) + params = {"count": 12} + return self._pagination_following(endpoint, params) + def user_saved(self): endpoint = "/v1/feed/saved/posts/" params = {"count": 50} @@ -960,6 +982,20 @@ class InstagramRestAPI(): return extr._update_cursor(None) params["max_id"] = extr._update_cursor(data["next_max_id"]) + def _pagination_following(self, endpoint, params): + extr = self.extractor + params["max_id"] = text.parse_int(extr._init_cursor()) + + while True: + data = self._call(endpoint, params=params) + + yield from data["users"] + + if len(data["users"]) < params["count"]: + return extr._update_cursor(None) + params["max_id"] = extr._update_cursor( + params["max_id"] + params["count"]) + class InstagramGraphqlAPI(): From d12a5e440a0e1c4a5a00706ee6da289a842f75fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 24 Aug 2023 15:01:26 +0200 Subject: [PATCH 07/53] update docs/supportedsites --- docs/supportedsites.md | 30 +++++++++++++------------- gallery_dl/extractor/imagehosts.py | 34 ++++++++++++++++++------------ scripts/supportedsites.py | 2 +- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index e095743f..676ba938 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -88,7 +88,7 @@ Consider all sites to be NSFW unless otherwise known. ArtStation https://www.artstation.com/ - Albums, Artwork Listings, Challenges, individual Images, Likes, Search Results, User Profiles + Albums, Artwork Listings, Challenges, Followed Users, individual Images, Likes, Search Results, User Profiles @@ -148,7 +148,7 @@ Consider all sites to be NSFW unless otherwise known. DeviantArt https://www.deviantart.com/ - Collections, Deviations, Favorites, Folders, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches + Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches OAuth @@ -226,7 +226,7 @@ Consider all sites to be NSFW unless otherwise known. Fur Affinity https://www.furaffinity.net/ - Favorites, Galleries, Posts, Scraps, Search Results, User Profiles + Favorites, Followed Users, Galleries, Posts, Scraps, Search Results, User Profiles Cookies @@ -357,7 +357,7 @@ Consider all sites to be NSFW unless otherwise known. Imagevenue - https://imagevenue.com/ + https://www.imagevenue.com/ individual Images @@ -406,13 +406,13 @@ Consider all sites to be NSFW unless otherwise known. Inkbunny https://inkbunny.net/ - Favorites, Pools, Posts, Search Results, User Profiles + Favorites, Followed Users, Pools, Posts, Search Results, User Profiles Supported Instagram https://www.instagram.com/ - Avatars, Collections, Guides, Highlights, Posts, Reels, Saved Posts, Stories, Tag Searches, Tagged Posts, User Profiles + Avatars, Collections, Followed Users, Guides, Highlights, Posts, Reels, Saved Posts, Stories, Tag Searches, Tagged Posts, User Profiles Cookies @@ -574,7 +574,7 @@ Consider all sites to be NSFW unless otherwise known. Newgrounds https://www.newgrounds.com/ - Art, Audio, Favorites, Games, individual Images, Media Files, Movies, Search Results, User Profiles + Art, Audio, Favorites, Followed Users, Games, individual Images, Media Files, Movies, Search Results, User Profiles Supported @@ -778,7 +778,7 @@ Consider all sites to be NSFW unless otherwise known. Skeb https://skeb.jp/ - Posts, Search Results, User Profiles + Followed Users, Posts, Search Results, User Profiles @@ -874,7 +874,7 @@ Consider all sites to be NSFW unless otherwise known. Twitter https://twitter.com/ - Avatars, Backgrounds, Bookmarks, Events, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles + Avatars, Backgrounds, Bookmarks, Events, Followed Users, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles Supported @@ -1160,19 +1160,19 @@ Consider all sites to be NSFW unless otherwise known. Misskey.io https://misskey.io/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles Lesbian.energy https://lesbian.energy/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles Sushi.ski https://sushi.ski/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles @@ -1472,19 +1472,19 @@ Consider all sites to be NSFW unless otherwise known. mastodon.social https://mastodon.social/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth Pawoo https://pawoo.net/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth baraag https://baraag.net/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 8ef51b0a..5c7e42e9 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -265,7 +265,7 @@ class PixhostImageExtractor(ImagehostImageExtractor): category = "pixhost" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" r"/show/\d+/(\d+)_[^/?#]+)") - test = ("http://pixhost.to/show/190/130327671_test-.png", { + test = ("https://pixhost.to/show/190/130327671_test-.png", { "url": "4e5470dcf6513944773044d40d883221bbc46cff", "keyword": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", "content": "0c8768055e4e20e7c7259608b67799171b691140", @@ -300,13 +300,18 @@ class PixhostGalleryExtractor(ImagehostImageExtractor): class PostimgImageExtractor(ImagehostImageExtractor): """Extractor for single images from postimages.org""" category = "postimg" - pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" - r"/(?!gallery/)(?:image/)?([^/?#]+)/?)") - test = ("https://postimg.cc/Wtn2b3hC", { - "url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", - "keyword": "2d05808d04e4e83e33200db83521af06e3147a84", - "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", - }) + pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" + r"\.(?:cc|org)/(?!gallery/)(?:image/)?([^/?#]+)/?)") + test = ( + ("https://postimages.org/Wtn2b3hC"), + ("https://www.postimages.org/Wtn2b3hC"), + ("https://pixxxels.cc/Wtn2b3hC"), + ("https://postimg.cc/Wtn2b3hC", { + "url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", + "keyword": "2d05808d04e4e83e33200db83521af06e3147a84", + "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", + }), + ) def get_info(self, page): pos = page.index(' id="download"') @@ -385,11 +390,14 @@ class FappicImageExtractor(ImagehostImageExtractor): """Extractor for single images from fappic.com""" category = "fappic" pattern = r"(?:https?://)?((?:www\.)?fappic\.com/(\w+)/[^/?#]+)" - test = ("https://www.fappic.com/98wxqcklyh8k/test.png", { - "pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", - "keyword": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + test = ( + ("https://fappic.com/98wxqcklyh8k/test.png"), + ("https://www.fappic.com/98wxqcklyh8k/test.png", { + "pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", + "keyword": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", + "content": "0c8768055e4e20e7c7259608b67799171b691140", + }), + ) def get_info(self, page): url , pos = text.extract(page, ' Date: Thu, 24 Aug 2023 15:23:38 +0200 Subject: [PATCH 08/53] [reddit] download preview for 404ed imgur links (#4322) This is a pretty ugly hack as the internal infrastructure doesn't really support switching from external URL to regular download in case the former fails, but it kind of works ... Can be disabled by setting 'reddit.fallback' to 'false'. --- gallery_dl/extractor/reddit.py | 11 +++++++++++ gallery_dl/job.py | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 05da7f4a..55a7f65f 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -108,7 +108,11 @@ class RedditExtractor(Extractor): if match: extra.append(match.group(1)) elif not match_user(url) and not match_subreddit(url): + if "preview" in data: + data["_fallback"] = self._previews(data) yield Message.Queue, text.unescape(url), data + if "_fallback" in data: + del data["_fallback"] if not extra or depth == max_depth: return @@ -165,6 +169,13 @@ class RedditExtractor(Extractor): submission["_ytdl_extra"] = {"title": submission["title"]} return submission["url"] + def _previews(self, post): + try: + for image in post["preview"]["images"]: + yield image["source"]["url"] + except Exception as exc: + self.log.debug("%s: %s", exc.__class__.__name__, exc) + class RedditSubredditExtractor(RedditExtractor): """Extractor for URLs from subreddits on reddit.com""" diff --git a/gallery_dl/job.py b/gallery_dl/job.py index af924151..5adb030c 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -388,10 +388,23 @@ class DownloadJob(Job): try: if pextr.config("parent-skip"): job._skipcnt = self._skipcnt - self.status |= job.run() + status = job.run() self._skipcnt = job._skipcnt else: - self.status |= job.run() + status = job.run() + + if status: + self.status |= status + if "_fallback" in kwdict and self.fallback: + fallback = kwdict["_fallback"] = \ + iter(kwdict["_fallback"]) + try: + url = next(fallback) + except StopIteration: + pass + else: + text.nameext_from_url(url, kwdict) + self.handle_url(url, kwdict) break except exception.RestartExtraction: pass From 06aaedded50b9736d6883849c5226d5bcba3b5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 28 Aug 2023 16:31:57 +0200 Subject: [PATCH 09/53] [twitter] extract 'source' metadata (#4459) --- gallery_dl/extractor/twitter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 34ec4162..e299abde 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -277,6 +277,8 @@ class TwitterExtractor(Extractor): else: note = None + source = tweet["source"] + if "legacy" in tweet: tweet = tweet["legacy"] @@ -303,6 +305,7 @@ class TwitterExtractor(Extractor): "author" : author, "user" : self._user or author, "lang" : tweet["lang"], + "source" : text.extr(source, ">", "<"), "favorite_count": tget("favorite_count"), "quote_count" : tget("quote_count"), "reply_count" : tget("reply_count"), From fd65f27ede916bb6bbc83fb56877a6eb9d9f3288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 28 Aug 2023 17:17:03 +0200 Subject: [PATCH 10/53] [reddit] fix 'preview.redd.it' URLs (#4470) --- gallery_dl/extractor/reddit.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 55a7f65f..39e98ba5 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -65,7 +65,10 @@ class RedditExtractor(Extractor): media = submission url = media["url"] - if url and url.startswith("https://i.redd.it/"): + if url and url.startswith(( + "https://i.redd.it/", + "https://preview.redd.it/", + )): text.nameext_from_url(url, submission) yield Message.Url, url, submission @@ -309,6 +312,11 @@ class RedditSubmissionExtractor(RedditExtractor): ("https://www.reddit.com/r/kittengifs/comments/12m0b8d", { "pattern": r"ytdl:https://v\.redd\.it/cvabpjacrvta1", }), + # preview.redd.it (#4470) + ("https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", { + "pattern": r"https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp" + r"&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", + }), ("https://old.reddit.com/r/lavaporn/comments/2a00np/"), ("https://np.reddit.com/r/lavaporn/comments/2a00np/"), ("https://m.reddit.com/r/lavaporn/comments/2a00np/"), From 7defb24e1ea1e73260f1e78e7b3d10916d168493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 28 Aug 2023 18:05:31 +0200 Subject: [PATCH 11/53] [reddit] provide video previews if available (#4322) --- gallery_dl/extractor/reddit.py | 10 ++++++++++ gallery_dl/job.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 39e98ba5..36708f1a 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -173,6 +173,16 @@ class RedditExtractor(Extractor): return submission["url"] def _previews(self, post): + try: + if "reddit_video_preview" in post["preview"]: + video = post["preview"]["reddit_video_preview"] + if "dash_url" in video: + yield "ytdl:" + video["dash_url"] + if "hls_url" in video: + yield "ytdl:" + video["hls_url"] + except Exception as exc: + self.log.debug("%s: %s", exc.__class__.__name__, exc) + try: for image in post["preview"]["images"]: yield image["source"]["url"] diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 5adb030c..e651d975 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -404,6 +404,8 @@ class DownloadJob(Job): pass else: text.nameext_from_url(url, kwdict) + if url.startswith("ytdl:"): + kwdict["extension"] = "" self.handle_url(url, kwdict) break except exception.RestartExtraction: From ba842981af47e830b16033e1a3c715e0ddf8c19a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 29 Aug 2023 12:06:30 +0200 Subject: [PATCH 12/53] [imagevenue] fix extraction (#4473) --- gallery_dl/extractor/imagehosts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 5c7e42e9..0f782e8f 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -209,6 +209,8 @@ class ImagevenueImageExtractor(ImagehostImageExtractor): def get_info(self, page): pos = page.index('class="card-body') url, pos = text.extract(page, ' Date: Tue, 29 Aug 2023 19:34:27 +0200 Subject: [PATCH 13/53] [pornhub] add 'gif' support (#4463) --- docs/supportedsites.md | 2 +- gallery_dl/extractor/pornhub.py | 158 ++++++++++++++++++++++++++------ scripts/supportedsites.py | 3 + 3 files changed, 133 insertions(+), 30 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 676ba938..02b55230 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -694,7 +694,7 @@ Consider all sites to be NSFW unless otherwise known. Pornhub https://www.pornhub.com/ - Galleries, User Profiles + Galleries, Gifs, Photos, User Profiles diff --git a/gallery_dl/extractor/pornhub.py b/gallery_dl/extractor/pornhub.py index d3619da6..6cb2063b 100644 --- a/gallery_dl/extractor/pornhub.py +++ b/gallery_dl/extractor/pornhub.py @@ -19,6 +19,35 @@ class PornhubExtractor(Extractor): category = "pornhub" root = "https://www.pornhub.com" + def _init(self): + self.cookies.set( + "accessAgeDisclaimerPH", "1", domain=".pornhub.com") + + def _pagination(self, user, path): + if "/" not in path: + path += "/public" + + url = "{}/{}/{}/ajax".format(self.root, user, path) + params = {"page": 1} + headers = { + "Referer": url[:-5], + "X-Requested-With": "XMLHttpRequest", + } + + while True: + response = self.request( + url, method="POST", headers=headers, params=params, + allow_redirects=False) + + if 300 <= response.status_code < 400: + url = "{}{}/{}/ajax".format( + self.root, response.headers["location"], path) + continue + + yield response.text + + params["page"] += 1 + class PornhubGalleryExtractor(PornhubExtractor): """Extractor for image galleries on pornhub.com""" @@ -58,9 +87,6 @@ class PornhubGalleryExtractor(PornhubExtractor): self._first = None def items(self): - self.cookies.set( - "accessAgeDisclaimerPH", "1", domain=".pornhub.com") - data = self.metadata() yield Message.Directory, data for num, image in enumerate(self.images(), 1): @@ -116,17 +142,83 @@ class PornhubGalleryExtractor(PornhubExtractor): return +class PornhubGifExtractor(PornhubExtractor): + """Extractor for pornhub.com gifs""" + subcategory = "gif" + directory_fmt = ("{category}", "{user}", "gifs") + filename_fmt = "{id} {title}.{extension}" + archive_fmt = "{id}" + pattern = BASE_PATTERN + r"/gif/(\d+)" + test = ( + ("https://www.pornhub.com/gif/33643461", { + "pattern": r"https://\w+\.phncdn\.com/pics/gifs" + r"/033/643/461/33643461a\.webm", + "keyword": { + "date": "dt:2020-10-31 00:00:00", + "extension": "webm", + "filename": "33643461a", + "id": "33643461", + "tags": ["big boobs", "lana rhoades"], + "title": "Big boobs", + "url": str, + "user": "Lana Rhoades", + }, + }), + ) + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.gallery_id = match.group(1) + + def items(self): + url = "{}/gif/{}".format(self.root, self.gallery_id) + extr = text.extract_from(self.request(url).text) + + gif = { + "id" : self.gallery_id, + "tags" : extr("data-context-tag='", "'").split(","), + "title": extr('"name": "', '"'), + "url" : extr('"contentUrl": "', '"'), + "date" : text.parse_datetime( + extr('"uploadDate": "', '"'), "%Y-%m-%d"), + "user" : extr('data-mxptext="', '"'), + } + + yield Message.Directory, gif + yield Message.Url, gif["url"], text.nameext_from_url(gif["url"], gif) + + class PornhubUserExtractor(PornhubExtractor): - """Extractor for all galleries of a pornhub user""" + """Extractor for a pornhub user""" subcategory = "user" - pattern = (BASE_PATTERN + r"/(users|model|pornstar)/([^/?#]+)" - "(?:/photos(?:/(public|private|favorites))?)?/?$") + pattern = BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)/?$" + test = ("https://www.pornhub.com/pornstar/danika-mori",) + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.user = match.group(1) + + def initialize(self): + pass + + def items(self): + base = "{}/{}/".format(self.root, self.user) + return self._dispatch_extractors(( + (PornhubPhotosExtractor, base + "photos"), + (PornhubGifsExtractor , base + "gifs"), + ), ("photos",)) + + +class PornhubPhotosExtractor(PornhubExtractor): + """Extractor for all galleries of a pornhub user""" + subcategory = "photos" + pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" + "/(photos(?:/[^/?#]+)?)") test = ( ("https://www.pornhub.com/pornstar/danika-mori/photos", { "pattern": PornhubGalleryExtractor.pattern, "count": ">= 6", }), - ("https://www.pornhub.com/users/flyings0l0/"), ("https://www.pornhub.com/users/flyings0l0/photos/public"), ("https://www.pornhub.com/users/flyings0l0/photos/private"), ("https://www.pornhub.com/users/flyings0l0/photos/favorites"), @@ -135,33 +227,41 @@ class PornhubUserExtractor(PornhubExtractor): def __init__(self, match): PornhubExtractor.__init__(self, match) - self.type, self.user, self.cat = match.groups() + self.user, self.path = match.groups() def items(self): - url = "{}/{}/{}/photos/{}/ajax".format( - self.root, self.type, self.user, self.cat or "public") - params = {"page": 1} - headers = { - "Referer": url[:-5], - "X-Requested-With": "XMLHttpRequest", - } - data = {"_extractor": PornhubGalleryExtractor} - while True: - response = self.request( - url, method="POST", headers=headers, params=params, - allow_redirects=False) - - if 300 <= response.status_code < 400: - url = "{}{}/photos/{}/ajax".format( - self.root, response.headers["location"], - self.cat or "public") - continue - + for page in self._pagination(self.user, self.path): gid = None - for gid in text.extract_iter(response.text, 'id="albumphoto', '"'): + for gid in text.extract_iter(page, 'id="albumphoto', '"'): yield Message.Queue, self.root + "/album/" + gid, data if gid is None: return - params["page"] += 1 + +class PornhubGifsExtractor(PornhubExtractor): + """Extractor for a pornhub user's gifs""" + subcategory = "gifs" + pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" + "/(gifs(?:/[^/?#]+)?)") + test = ( + ("https://www.pornhub.com/pornstar/danika-mori/gifs", { + "pattern": PornhubGifExtractor.pattern, + "count": ">= 42", + }), + ("https://www.pornhub.com/users/flyings0l0/gifs"), + ("https://www.pornhub.com/model/bossgirl/gifs/video"), + ) + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.user, self.path = match.groups() + + def items(self): + data = {"_extractor": PornhubGifExtractor} + for page in self._pagination(self.user, self.path): + gid = None + for gid in text.extract_iter(page, 'id="gif', '"'): + yield Message.Queue, self.root + "/gif/" + gid, data + if gid is None: + return diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 68925caa..9077a036 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -224,6 +224,9 @@ SUBCATEGORY_MAP = { "sketch": "Sketch", "work": "individual Images", }, + "pornhub": { + "gifs": "", + }, "reddit": { "home": "Home Feed", }, From 28798594e8dd165909ffd6d44578d7a109aae2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 30 Aug 2023 18:35:03 +0200 Subject: [PATCH 14/53] [gfycat] update pagination logic (#4479) Some searches do not use cursor based pagination but an offset based one. --- gallery_dl/extractor/gfycat.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gallery_dl/extractor/gfycat.py b/gallery_dl/extractor/gfycat.py index 53ef1180..a95dab85 100644 --- a/gallery_dl/extractor/gfycat.py +++ b/gallery_dl/extractor/gfycat.py @@ -248,7 +248,7 @@ class GfycatAPI(): def search(self, query): endpoint = "/v1/gfycats/search" - params = {"search_text": query, "count": 150} + params = {"search_text": query, "count": 100} return self._pagination(endpoint, params) def user(self, user): @@ -301,6 +301,12 @@ class GfycatAPI(): data = self._call(endpoint, params) yield from data[key] - if not data["cursor"]: + if data["cursor"]: + params["cursor"] = data["cursor"] + elif "cursor" in params: return - params["cursor"] = data["cursor"] + else: + start = params.get("start", 0) + if start + len(data[key]) >= data.get("found", 0): + return + params["start"] = start + params["count"] From d3046561d43263d92d0288f9a23c0dacc1de3c06 Mon Sep 17 00:00:00 2001 From: johnsmith1202gmail <143646601+johnsmith1202gmail@users.noreply.github.com> Date: Wed, 30 Aug 2023 18:21:06 +0000 Subject: [PATCH 15/53] continue downloading when the item is made private on pixiv --- gallery_dl/extractor/pixiv.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index ffe8030f..1265a945 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -1210,6 +1210,10 @@ class PixivAppAPI(): self.extractor.wait(seconds=300) continue + if response.status_code == 403 \ + and "Artist has made their work private" in (error.get("message") or "").lower(): + raise exception.NotFoundError() + raise exception.StopExtraction("API request failed: %s", error) def _pagination(self, endpoint, params, key="illusts"): From c7e31b272492ba6485facef92367a85bc8ae731e Mon Sep 17 00:00:00 2001 From: johnsmith1202gmail <143646601+johnsmith1202gmail@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:42:31 -0700 Subject: [PATCH 16/53] Update pixiv.py --- gallery_dl/extractor/pixiv.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 1265a945..162f27a8 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -863,9 +863,14 @@ class PixivNovelExtractor(PixivExtractor): novel["suffix"] = "" yield Message.Directory, novel + + try: + content = self.api.novel_text(novel["id"])["novel_text"] + except Exception: + self.log.warning("Unable to download novel %s", novel["id"]) + continue novel["extension"] = "txt" - content = self.api.novel_text(novel["id"])["novel_text"] yield Message.Url, "text:" + content, novel if embeds: @@ -1210,10 +1215,6 @@ class PixivAppAPI(): self.extractor.wait(seconds=300) continue - if response.status_code == 403 \ - and "Artist has made their work private" in (error.get("message") or "").lower(): - raise exception.NotFoundError() - raise exception.StopExtraction("API request failed: %s", error) def _pagination(self, endpoint, params, key="illusts"): From 8c477f7146be81afe5de5686cef477d4cf8fb87c Mon Sep 17 00:00:00 2001 From: Cisney-Gassai <55044372+Cisney-Gassai@users.noreply.github.com> Date: Sat, 2 Sep 2023 01:45:59 -0600 Subject: [PATCH 17/53] [bunkr] Fixes media-files-pizza.bunkr.ru failed to resolve. --- gallery_dl/extractor/bunkr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index d7b9b0c8..03b84288 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -15,6 +15,7 @@ from urllib.parse import urlsplit, urlunsplit MEDIA_DOMAIN_OVERRIDES = { "cdn9.bunkr.ru" : "c9.bunkr.ru", "cdn12.bunkr.ru": "media-files12.bunkr.la", + "cdn-pizza.bunkr.ru": "pizza.bunkr.ru", } CDN_HOSTED_EXTENSIONS = ( From a05821f8b4dbfa38618c4a10cc93db91b92741bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 2 Sep 2023 12:57:12 +0200 Subject: [PATCH 18/53] [hiperdex] fix 'manga' metadata remove trailing ' Manga' --- gallery_dl/extractor/hiperdex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index 22fe7f6c..70c586b0 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -31,7 +31,7 @@ class HiperdexBase(): return { "manga" : text.unescape(extr( - "", "<").rpartition(" - ")[0].strip()), + "<title>", "<").rpartition(" Manga - ")[0].strip()), "url" : text.unescape(extr( 'property="og:url" content="', '"')), "score" : text.parse_float(extr( From 0b6e5b816186e94fb2ec26731de1d18645c5b56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 2 Sep 2023 12:51:57 +0200 Subject: [PATCH 19/53] [hiperdex] send Referer headers during file downloads (#4490) --- gallery_dl/extractor/hiperdex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index 70c586b0..96f7cc11 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -22,6 +22,9 @@ class HiperdexBase(): category = "hiperdex" root = "https://hiperdex.com" + def _init(self): + self.session.headers["Referer"] = self.root + "/" + @memcache(keyarg=1) def manga_data(self, manga, page=None): if not page: From 8259a5abe47778861bc01de03433368f6e1a4607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 2 Sep 2023 13:03:11 +0200 Subject: [PATCH 20/53] flake8 --- gallery_dl/extractor/pixiv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 162f27a8..fc5f39c4 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -863,7 +863,7 @@ class PixivNovelExtractor(PixivExtractor): novel["suffix"] = "" yield Message.Directory, novel - + try: content = self.api.novel_text(novel["id"])["novel_text"] except Exception: From ceb59e176f3170cd054b02b6309d4f93966dc237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 2 Sep 2023 22:11:57 +0200 Subject: [PATCH 21/53] fix default Firefox user agent string note to self: do not trust some random third-party website --- docs/configuration.rst | 2 +- docs/gallery-dl.conf | 2 +- gallery_dl/extractor/common.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5f8ac6eb..ddb028ef 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -534,7 +534,7 @@ extractor.*.user-agent Type ``string`` Default - ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0"`` + ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0"`` Description User-Agent header value to be used for HTTP requests. diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index b5efc734..6f397e68 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -10,7 +10,7 @@ "proxy": null, "skip": true, - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", "retries": 4, "timeout": 30.0, "verify": true, diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 660f9551..6f48e49d 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -299,7 +299,7 @@ class Extractor(): useragent = self.config("user-agent") if useragent is None: useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " - "rv:115.0) Gecko/20100101 Firefox/115.0") + "rv:109.0) Gecko/20100101 Firefox/115.0") elif useragent == "browser": useragent = _browser_useragent() headers["User-Agent"] = useragent @@ -831,8 +831,8 @@ _browser_cookies = {} HTTP_HEADERS = { "firefox": ( - ("User-Agent", "Mozilla/5.0 ({}; rv:115.0) " - "Gecko/20100101 Firefox/115.0"), + ("User-Agent", "Mozilla/5.0 ({}; " + "rv:109.0) Gecko/20100101 Firefox/115.0"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," "image/avif,image/webp,*/*;q=0.8"), ("Accept-Language", "en-US,en;q=0.5"), From 23bac772f26cf19ea6ecb50133c7ba2a2f582f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sun, 3 Sep 2023 15:03:47 +0200 Subject: [PATCH 22/53] [jpgfish] update domain to 'jpg1.su' (#4494) --- docs/supportedsites.md | 2 +- gallery_dl/extractor/jpgfish.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 02b55230..c50e416d 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -435,7 +435,7 @@ Consider all sites to be NSFW unless otherwise known. </tr> <tr> <td>JPG Fish</td> - <td>https://jpeg.pet/</td> + <td>https://jpg1.su/</td> <td>Albums, individual Images, User Profiles</td> <td></td> </tr> diff --git a/gallery_dl/extractor/jpgfish.py b/gallery_dl/extractor/jpgfish.py index 39208e5a..a63acb55 100644 --- a/gallery_dl/extractor/jpgfish.py +++ b/gallery_dl/extractor/jpgfish.py @@ -4,18 +4,18 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for https://jpeg.pet/""" +"""Extractors for https://jpg1.su/""" from .common import Extractor, Message from .. import text -BASE_PATTERN = r"(?:https?://)?jpe?g\.(?:pet|fish(?:ing)?|church)" +BASE_PATTERN = r"(?:https?://)?jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)" class JpgfishExtractor(Extractor): """Base class for jpgfish extractors""" category = "jpgfish" - root = "https://jpeg.pet" + root = "https://jpg1.su" directory_fmt = ("{category}", "{user}", "{album}",) archive_fmt = "{id}" @@ -36,7 +36,7 @@ class JpgfishImageExtractor(JpgfishExtractor): subcategory = "image" pattern = BASE_PATTERN + r"/img/((?:[^/?#]+\.)?(\w+))" test = ( - ("https://jpeg.pet/img/funnymeme.LecXGS", { + ("https://jpg1.su/img/funnymeme.LecXGS", { "pattern": r"https://simp3\.jpg\.church/images/funnymeme\.jpg", "content": "098e5e9b17ad634358426e0ffd1c93871474d13c", "keyword": { @@ -52,6 +52,7 @@ class JpgfishImageExtractor(JpgfishExtractor): "pattern": r"https://simp2\.jpg\.church/hannahowo_00457\.jpg", "keyword": {"album": "401-500"}, }), + ("https://jpeg.pet/img/funnymeme.LecXGS"), ("https://jpg.pet/img/funnymeme.LecXGS"), ("https://jpg.fishing/img/funnymeme.LecXGS"), ("https://jpg.fish/img/funnymeme.LecXGS"), @@ -84,7 +85,7 @@ class JpgfishAlbumExtractor(JpgfishExtractor): subcategory = "album" pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?" test = ( - ("https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", { + ("https://jpg1.su/album/CDilP/?sort=date_desc&page=1", { "count": 2, }), ("https://jpg.fishing/a/gunggingnsk.N9OOI", { @@ -96,6 +97,7 @@ class JpgfishAlbumExtractor(JpgfishExtractor): ("https://jpg.church/a/hannahowo.aNTdH/sub", { "count": 606, }), + ("https://jpeg.pet/album/CDilP/?sort=date_desc&page=1"), ("https://jpg.pet/album/CDilP/?sort=date_desc&page=1"), ) @@ -122,12 +124,13 @@ class JpgfishUserExtractor(JpgfishExtractor): subcategory = "user" pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?" test = ( - ("https://jpeg.pet/exearco", { + ("https://jpg1.su/exearco", { "count": 3, }), ("https://jpg.church/exearco/albums", { "count": 1, }), + ("https://jpeg.pet/exearco"), ("https://jpg.pet/exearco"), ("https://jpg.fishing/exearco"), ("https://jpg.fish/exearco"), From 6eca1fab9ba05822a4a349640c0a617f82c4e915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sun, 3 Sep 2023 15:39:02 +0200 Subject: [PATCH 23/53] [gelbooru_v02] support 'xbooru.com' (#4493) --- docs/supportedsites.md | 6 ++++++ gallery_dl/extractor/gelbooru_v02.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index c50e416d..93eb3e36 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1121,6 +1121,12 @@ Consider all sites to be NSFW unless otherwise known. <td>Favorites, Pools, Posts, Tag Searches</td> <td></td> </tr> +<tr> + <td>Xbooru</td> + <td>https://xbooru.com/</td> + <td>Favorites, Pools, Posts, Tag Searches</td> + <td></td> +</tr> <tr> <td colspan="4"><strong>jschan Imageboards</strong></td> diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index 1ef78efd..28bc799e 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -183,6 +183,10 @@ INSTANCES = { "root": "https://hypnohub.net", "pattern": r"hypnohub\.net", }, + "xbooru": { + "root": "https://xbooru.com", + "pattern": r"xbooru\.com", + }, } BASE_PATTERN = GelbooruV02Extractor.update(INSTANCES) @@ -213,6 +217,9 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor): ("https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", { "url": "fe662b86d38c331fcac9c62af100167d404937dc", }), + ("https://xbooru.com/index.php?page=post&s=list&tags=konoyan", { + "count": 11, + }), ) def __init__(self, match): @@ -246,6 +253,10 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor): "url": "d314826280073441a2da609f70ee814d1f4b9407", "count": 3, }), + ("https://xbooru.com/index.php?page=pool&s=show&id=757", { + "url": "ceeac56c179ec72301bd0b6add6355a138fdea01", + "count": 5, + }), ) def __init__(self, match): @@ -314,6 +325,9 @@ class GelbooruV02FavoriteExtractor(GelbooruV02Extractor): ("https://hypnohub.net/index.php?page=favorites&s=view&id=43546", { "count": 3, }), + ("https://xbooru.com/index.php?page=favorites&s=view&id=45206", { + "count": 4, + }), ) def __init__(self, match): @@ -440,6 +454,11 @@ class GelbooruV02PostExtractor(GelbooruV02Extractor): "url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", "content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", }), + ("https://xbooru.com/index.php?page=post&s=view&id=1025649", { + "pattern": r"https://img\.xbooru\.com/images/444" + r"/f3eda549ad8b9db244ac335c7406c92f\.jpeg", + "content": "086668afd445438d491ecc11cee3ac69b4d65530", + }), ) def __init__(self, match): From f2de70f2545e413def0418c55cc4dd7adcd19a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 4 Sep 2023 18:27:11 +0200 Subject: [PATCH 24/53] [gfycat] remove module --- docs/configuration.rst | 28 +-- docs/gallery-dl-example.conf | 2 +- docs/gallery-dl.conf | 4 - docs/supportedsites.md | 6 - gallery_dl/extractor/__init__.py | 1 - gallery_dl/extractor/gfycat.py | 312 ------------------------------- scripts/supportedsites.py | 4 - test/test_results.py | 2 +- 8 files changed, 4 insertions(+), 355 deletions(-) delete mode 100644 gallery_dl/extractor/gfycat.py diff --git a/docs/configuration.rst b/docs/configuration.rst index ddb028ef..819dc959 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -390,7 +390,6 @@ Description * ``e621`` (*) * ``e926`` (*) * ``exhentai`` - * ``gfycat`` * ``idolcomplex`` * ``imgbb`` * ``inkbunny`` @@ -714,7 +713,7 @@ Type Default ``["oauth", "recursive", "test"]`` + current extractor category Example - ``["imgur", "gfycat:user", "*:image"]`` + ``["imgur", "redgifs:user", "*:image"]`` Description A list of extractor identifiers to ignore (or allow) when spawning child extractors for unknown URLs, @@ -723,7 +722,7 @@ Description Each identifier can be * A category or basecategory name (``"imgur"``, ``"mastodon"``) - * | A (base)category-subcategory pair, where both names are separated by a colon (``"gfycat:user"``). + * | A (base)category-subcategory pair, where both names are separated by a colon (``"redgifs:user"``). | Both names can be a `*` or left empty, matching all possible names (``"*:image"``, ``":user"``). Note: Any ``blacklist`` setting will automatically include @@ -1706,29 +1705,6 @@ Description even ones without a ``generic:`` prefix. -extractor.gfycat.format ------------------------ -Type - * ``string`` - * ``list`` of ``strings`` -Default - ``["mp4", "webm", "mobile", "gif"]`` -Description - List of names of the preferred animation format, which can be - ``"mp4"``, - ``"webm"``, - ``"mobile"``, - ``"gif"``, or - ``"webp"``. - - If a selected format is not available, the next one in the list will be - tried until an available format is found. - - If the format is given as ``string``, it will be extended with - ``["mp4", "webm", "mobile", "gif"]``. Use a list with one element to - restrict it to only one possible format. - - extractor.gofile.api-token -------------------------- Type diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf index da386dd6..c3f80493 100644 --- a/docs/gallery-dl-example.conf +++ b/docs/gallery-dl-example.conf @@ -165,7 +165,7 @@ "reddit": { "#": "only spawn child extractors for links to specific sites", - "whitelist": ["imgur", "redgifs", "gfycat"], + "whitelist": ["imgur", "redgifs"], "#": "put files from child extractors into the reddit directory", "parent-directory": true, diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 6f397e68..e0594cad 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -125,10 +125,6 @@ "api-key": null, "user-id": null }, - "gfycat": - { - "format": ["mp4", "webm", "mobile", "gif"] - }, "gofile": { "api-token": null, "website-token": null diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 93eb3e36..111cef65 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -247,12 +247,6 @@ Consider all sites to be NSFW unless otherwise known. <td>Favorites, Pools, Posts, Redirects, Tag Searches</td> <td></td> </tr> -<tr> - <td>Gfycat</td> - <td>https://gfycat.com/</td> - <td>Collections, individual Images, Search Results, User Profiles</td> - <td>Supported</td> -</tr> <tr> <td>Gofile</td> <td>https://gofile.io/</td> diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index fa56bfb4..3abe74b6 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -49,7 +49,6 @@ modules = [ "gelbooru", "gelbooru_v01", "gelbooru_v02", - "gfycat", "gofile", "hbrowse", "hentai2read", diff --git a/gallery_dl/extractor/gfycat.py b/gallery_dl/extractor/gfycat.py deleted file mode 100644 index a95dab85..00000000 --- a/gallery_dl/extractor/gfycat.py +++ /dev/null @@ -1,312 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2017-2023 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Extractors for https://gfycat.com/""" - -from .common import Extractor, Message -from .. import text, exception -from ..cache import cache - - -class GfycatExtractor(Extractor): - """Base class for gfycat extractors""" - category = "gfycat" - filename_fmt = "{category}_{gfyName}{title:?_//}.{extension}" - archive_fmt = "{gfyName}" - root = "https://gfycat.com" - - def __init__(self, match): - Extractor.__init__(self, match) - self.key = match.group(1).lower() - - def _init(self): - formats = self.config("format") - if formats is None: - formats = ("mp4", "webm", "mobile", "gif") - elif isinstance(formats, str): - formats = (formats, "mp4", "webm", "mobile", "gif") - self.formats = formats - - def items(self): - metadata = self.metadata() - for gfycat in self.gfycats(): - if "gfyName" not in gfycat: - self.log.warning("Skipping '%s' (malformed)", gfycat["gfyId"]) - continue - - url = self._process(gfycat) - if not url: - self.log.warning("Skipping '%s' (format not available)", - gfycat["gfyId"]) - continue - - gfycat.update(metadata) - yield Message.Directory, gfycat - yield Message.Url, url, gfycat - - def _process(self, gfycat): - gfycat["_fallback"] = formats = self._formats(gfycat) - gfycat["date"] = text.parse_timestamp(gfycat.get("createDate")) - return next(formats, None) - - def _formats(self, gfycat): - for fmt in self.formats: - key = fmt + "Url" - if key in gfycat: - url = gfycat[key] - if url.startswith("http:"): - url = "https" + url[4:] - gfycat["extension"] = url.rpartition(".")[2] - yield url - - def metadata(self): - return {} - - def gfycats(self): - return () - - -class GfycatUserExtractor(GfycatExtractor): - """Extractor for gfycat user profiles""" - subcategory = "user" - directory_fmt = ("{category}", "{username}") - pattern = r"(?:https?://)?gfycat\.com/@([^/?#]+)/?(?:$|\?|#)" - test = ("https://gfycat.com/@gretta", { - "pattern": r"https://giant\.gfycat\.com/[A-Za-z]+\.mp4", - "count": ">= 100", - }) - - def gfycats(self): - if self.key == "me": - return GfycatAPI(self).me() - return GfycatAPI(self).user(self.key) - - -class GfycatCollectionExtractor(GfycatExtractor): - """Extractor for a gfycat collection""" - subcategory = "collection" - directory_fmt = ("{category}", "{collection_owner}", - "{collection_name|collection_id}") - pattern = (r"(?:https?://)?gfycat\.com/@([^/?#]+)/collections" - r"/(\w+)(?:/([^/?#]+))?") - test = ("https://gfycat.com/@reactions/collections/nHgy2DtE/no-text", { - "pattern": r"https://\w+\.gfycat\.com/[A-Za-z]+\.mp4", - "count": ">= 100", - }) - - def __init__(self, match): - GfycatExtractor.__init__(self, match) - self.collection_id = match.group(2) - self.collection_name = match.group(3) - - def metadata(self): - return { - "collection_owner": self.key, - "collection_name" : self.collection_name, - "collection_id" : self.collection_id, - } - - def gfycats(self): - return GfycatAPI(self).collection(self.key, self.collection_id) - - -class GfycatCollectionsExtractor(GfycatExtractor): - """Extractor for a gfycat user's collections""" - subcategory = "collections" - pattern = r"(?:https?://)?gfycat\.com/@([^/?#]+)/collections/?(?:$|\?|#)" - test = ("https://gfycat.com/@sannahparker/collections", { - "pattern": GfycatCollectionExtractor.pattern, - "count": ">= 20", - }) - - def items(self): - for col in GfycatAPI(self).collections(self.key): - url = "https://gfycat.com/@{}/collections/{}/{}".format( - col["userId"], col["folderId"], col["linkText"]) - col["_extractor"] = GfycatCollectionExtractor - yield Message.Queue, url, col - - -class GfycatSearchExtractor(GfycatExtractor): - """Extractor for gfycat search results""" - subcategory = "search" - directory_fmt = ("{category}", "Search", "{search}") - pattern = r"(?:https?://)?gfycat\.com/gifs/search/([^/?#]+)" - test = ("https://gfycat.com/gifs/search/funny+animals", { - "pattern": r"https://\w+\.gfycat\.com/[A-Za-z]+\.mp4", - "archive": False, - "range": "100-300", - "count": "> 200", - }) - - def metadata(self): - self.key = text.unquote(self.key).replace("+", " ") - return {"search": self.key} - - def gfycats(self): - return GfycatAPI(self).search(self.key) - - -class GfycatImageExtractor(GfycatExtractor): - """Extractor for individual images from gfycat.com""" - subcategory = "image" - pattern = (r"(?:https?://)?(?:\w+\.)?gfycat\.com" - r"/(?:gifs/detail/|\w+/)?([A-Za-z]{8,})") - test = ( - ("https://gfycat.com/GrayGenerousCowrie", { - "url": "e0b5e1d7223108249b15c3c7898dd358dbfae045", - "content": "5786028e04b155baa20b87c5f4f77453cd5edc37", - "keyword": { - "gfyId": "graygenerouscowrie", - "gfyName": "GrayGenerousCowrie", - "gfyNumber": 755075459, - "title": "Bottom's up", - "username": "jackson3oh3", - "createDate": 1495884169, - "date": "dt:2017-05-27 11:22:49", - "md5": "a4796e05b0db9ba9ce5140145cd318aa", - "width": 400, - "height": 224, - "frameRate": 23.0, - "numFrames": 158.0, - "views": int, - }, - }), - (("https://thumbs.gfycat.com/SillyLameIsabellinewheatear" - "-size_restricted.gif"), { - "url": "13b32e6cc169d086577d7dd3fd36ee6cdbc02726", - }), - ("https://gfycat.com/detail/UnequaledHastyAnkole?tagname=aww", { - "url": "e24c9f69897fd223343782425a429c5cab6a768e", - }), - # retry 404'ed videos on redgifs (#874) - ("https://www.gfycat.com/foolishforkedabyssiniancat", { - "pattern": "https://redgifs.com/watch/foolishforkedabyssiniancat", - }), - # malformed API response (#902) - ("https://gfycat.com/illexcitablehairstreak", { - "count": 0, - }), - ("https://gfycat.com/gifs/detail/UnequaledHastyAnkole"), - ("https://gfycat.com/ifr/UnequaledHastyAnkole"), - ("https://gfycat.com/ru/UnequaledHastyAnkole"), - ) - - def items(self): - try: - gfycat = GfycatAPI(self).gfycat(self.key) - except exception.HttpError: - from .redgifs import RedgifsImageExtractor - url = "https://redgifs.com/watch/" + self.key - data = {"_extractor": RedgifsImageExtractor} - yield Message.Queue, url, data - else: - if "gfyName" not in gfycat: - self.log.warning("Skipping '%s' (malformed)", gfycat["gfyId"]) - return - url = self._process(gfycat) - if not url: - self.log.warning("Skipping '%s' (format not available)", - gfycat["gfyId"]) - return - yield Message.Directory, gfycat - yield Message.Url, url, gfycat - - -class GfycatAPI(): - API_ROOT = "https://api.gfycat.com" - - def __init__(self, extractor): - self.extractor = extractor - self.headers = {} - self.username, self.password = extractor._get_auth_info() - - def collection(self, user, collection): - endpoint = "/v1/users/{}/collections/{}/gfycats".format( - user, collection) - params = {"count": 100} - return self._pagination(endpoint, params) - - def collections(self, user): - endpoint = "/v1/users/{}/collections".format(user) - params = {"count": 100} - return self._pagination(endpoint, params, "gfyCollections") - - def gfycat(self, gfycat_id): - endpoint = "/v1/gfycats/" + gfycat_id - return self._call(endpoint)["gfyItem"] - - def me(self): - endpoint = "/v1/me/gfycats" - params = {"count": 100} - return self._pagination(endpoint, params) - - def search(self, query): - endpoint = "/v1/gfycats/search" - params = {"search_text": query, "count": 100} - return self._pagination(endpoint, params) - - def user(self, user): - endpoint = "/v1/users/{}/gfycats".format(user.lower()) - params = {"count": 100} - return self._pagination(endpoint, params) - - def authenticate(self): - self.headers["Authorization"] = \ - self._authenticate_impl(self.username, self.password) - - @cache(maxage=3600, keyarg=1) - def _authenticate_impl(self, username, password): - self.extractor.log.info("Logging in as %s", username) - - url = "https://weblogin.gfycat.com/oauth/webtoken" - headers = {"Origin": "https://gfycat.com"} - data = { - "access_key": "Anr96uuqt9EdamSCwK4txKPjMsf2" - "M95Rfa5FLLhPFucu8H5HTzeutyAa", - } - response = self.extractor.request( - url, method="POST", headers=headers, json=data).json() - - url = "https://weblogin.gfycat.com/oauth/weblogin" - headers["authorization"] = "Bearer " + response["access_token"] - data = { - "grant_type": "password", - "username" : username, - "password" : password, - } - response = self.extractor.request( - url, method="POST", headers=headers, json=data, fatal=None).json() - - if "errorMessage" in response: - raise exception.AuthenticationError( - response["errorMessage"]["description"]) - return "Bearer " + response["access_token"] - - def _call(self, endpoint, params=None): - if self.username: - self.authenticate() - - url = self.API_ROOT + endpoint - return self.extractor.request( - url, params=params, headers=self.headers).json() - - def _pagination(self, endpoint, params, key="gfycats"): - while True: - data = self._call(endpoint, params) - yield from data[key] - - if data["cursor"]: - params["cursor"] = data["cursor"] - elif "cursor" in params: - return - else: - start = params.get("start", 0) - if start + len(data[key]) >= data.get("found", 0): - return - params["start"] = start + params["count"] diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 9077a036..b8052164 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -178,9 +178,6 @@ SUBCATEGORY_MAP = { "fapello": { "path": "Videos, Trending Posts, Popular Videos, Top Models", }, - "gfycat": { - "collections": "", - }, "hentaifoundry": { "story": "", }, @@ -314,7 +311,6 @@ AUTH_MAP = { "fanbox" : _COOKIES, "fantia" : _COOKIES, "flickr" : _OAUTH, - "gfycat" : "Supported", "furaffinity" : _COOKIES, "horne" : "Required", "idolcomplex" : "Supported", diff --git a/test/test_results.py b/test/test_results.py index f434b2fc..a555b721 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -326,7 +326,7 @@ def setup_test_config(): "e621", "e926", "e6ai", "instagram", "twitter", "subscribestar", "deviantart", "inkbunny", "tapas", "pillowfort", "mangadex", - "vipergirls", "gfycat"): + "vipergirls"): config.set(("extractor", category), "username", None) config.set(("extractor", "mastodon.social"), "access-token", From 4c0b3d5dc5098438eafaac5b9ab18eea61a66575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 4 Sep 2023 18:28:43 +0200 Subject: [PATCH 25/53] [twitter] fix crash when 'sortIndex' is None (#4499) --- gallery_dl/extractor/twitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index e299abde..bf925f15 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -645,7 +645,7 @@ class TwitterLikesExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_liked"] = text.parse_timestamp( - (int(tweet["sortIndex"]) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata @@ -661,7 +661,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_bookmarked"] = text.parse_timestamp( - (int(tweet["sortIndex"]) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata From f8569872971d729fe1c0a826b180b0de5398489f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 4 Sep 2023 22:04:52 +0200 Subject: [PATCH 26/53] [subscribestar] fix preview detection (#4468) and show a warning message when posts contain previews --- gallery_dl/extractor/subscribestar.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py index a2e1388c..32c17d76 100644 --- a/gallery_dl/extractor/subscribestar.py +++ b/gallery_dl/extractor/subscribestar.py @@ -84,16 +84,16 @@ class SubscribestarExtractor(Extractor): if cookie.name.startswith("auth") } - @staticmethod - def _media_from_post(html): + def _media_from_post(self, html): media = [] gallery = text.extr(html, 'data-gallery="', '"') if gallery: - media.extend( - item for item in util.json_loads(text.unescape(gallery)) - if "/previews/" not in item["url"] - ) + for item in util.json_loads(text.unescape(gallery)): + if "/previews" in item["url"]: + self._warn_preview() + else: + media.append(item) attachments = text.extr( html, 'class="uploads-docs"', 'data-role="post-edit_form"') @@ -130,6 +130,10 @@ class SubscribestarExtractor(Extractor): date = text.parse_datetime(dt, "%B %d, %Y %I:%M %p") return date + def _warn_preview(self): + self.log.warning("Preview image detected") + self._warn_preview = util.noop + class SubscribestarUserExtractor(SubscribestarExtractor): """Extractor for media from a subscribestar user""" @@ -138,7 +142,8 @@ class SubscribestarUserExtractor(SubscribestarExtractor): test = ( ("https://www.subscribestar.com/subscribestar", { "count": ">= 20", - "pattern": r"https://\w+\.cloudfront\.net/uploads(_v2)?/users/11/", + "pattern": r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net" + r"/uploads(_v2)?/users/11/", "keyword": { "author_id": 11, "author_name": "subscribestar", From 93a7a89cf63e21fb60e7dbc8be9efdfd05328ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Tue, 5 Sep 2023 17:53:27 +0200 Subject: [PATCH 27/53] [formatter] use value of last alternative (#4492) fixes {fieldname|''} evaluating to the value of 'keywords-default' instead of an empty string --- gallery_dl/formatter.py | 5 +++-- test/test_formatter.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 98ea186f..6098fc61 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -182,9 +182,10 @@ class StringFormatter(): if obj: break except Exception: - pass + obj = None else: - obj = self.default + if obj is None: + obj = self.default return fmt(obj) return wrap diff --git a/test/test_formatter.py b/test/test_formatter.py index 0992f4ba..dbdccba1 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -340,6 +340,8 @@ class TestFormatter(unittest.TestCase): self._run_test("{'foobar'[:3]}", value) self._run_test("{z|'foo'}" , value) self._run_test("{z|''|'foo'}" , value) + self._run_test("{z|''}" , "") + self._run_test("{''|''}" , "") self._run_test("{_lit[foo]}" , value) self._run_test("{_lit[foo]!u}" , value.upper()) From 1d2b5d0c609610b8a60fec66e617659e90d5c848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 6 Sep 2023 18:16:09 +0200 Subject: [PATCH 28/53] update test comment positions always put them above the test they're referring to --- gallery_dl/extractor/deviantart.py | 3 +- gallery_dl/extractor/imagehosts.py | 9 +++-- gallery_dl/extractor/imgur.py | 63 ++++++++++++++++++++---------- gallery_dl/extractor/keenspot.py | 17 +++++--- gallery_dl/extractor/mangadex.py | 4 +- gallery_dl/extractor/pixnet.py | 6 +-- gallery_dl/extractor/reactor.py | 15 ++++--- gallery_dl/extractor/tapas.py | 5 ++- gallery_dl/extractor/tumblr.py | 24 ++++++++---- 9 files changed, 95 insertions(+), 51 deletions(-) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 3302afbe..1cca967e 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -693,8 +693,9 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): archive_fmt = "f_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" test = ( + # (#271) ("https://www.deviantart.com/h3813067/favourites/", { - "options": (("metadata", True), ("flat", False)), # issue #271 + "options": (("metadata", True), ("flat", False)), "count": 1, }), ("https://www.deviantart.com/h3813067/favourites/", { diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 0f782e8f..93dfa3e4 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -75,7 +75,8 @@ class ImxtoImageExtractor(ImagehostImageExtractor): pattern = (r"(?:https?://)?(?:www\.)?((?:imx\.to|img\.yt)" r"/(?:i/|img-)(\w+)(\.html)?)") test = ( - ("https://imx.to/i/1qdeva", { # new-style URL + # new-style URL + ("https://imx.to/i/1qdeva", { "url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130", "content": "0c8768055e4e20e7c7259608b67799171b691140", "keyword": { @@ -85,7 +86,8 @@ class ImxtoImageExtractor(ImagehostImageExtractor): "hash" : "94d56c599223c59f3feb71ea603484d1", }, }), - ("https://imx.to/img-57a2050547b97.html", { # old-style URL + # old-style URL + ("https://imx.to/img-57a2050547b97.html", { "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", "content": "54592f2635674c25677c6872db3709d343cdf92f", "keyword": { @@ -95,7 +97,8 @@ class ImxtoImageExtractor(ImagehostImageExtractor): "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", }, }), - ("https://img.yt/img-57a2050547b97.html", { # img.yt domain + # img.yt domain + ("https://img.yt/img-57a2050547b97.html", { "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", }), ("https://imx.to/img-57a2050547b98.html", { diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index ca9671c3..fd29a0e2 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -104,29 +104,42 @@ class ImgurImageExtractor(ImgurExtractor): "width" : 64, }, }), - ("http://imgur.com/0gybAXR", { # gifv/mp4 video + # gifv/mp4 video + ("http://imgur.com/0gybAXR", { "url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", "content": "a3c080e43f58f55243ab830569ba02309d59abfc", }), - ("https://imgur.com/XFfsmuC", { # missing title in API response (#467) + # missing title in API response (#467) + ("https://imgur.com/XFfsmuC", { "keyword": {"title": "Tears are a natural response to irritants"}, }), - ("https://imgur.com/1Nily2P", { # animated png + # animated png + ("https://imgur.com/1Nily2P", { "pattern": "https://i.imgur.com/1Nily2P.png", }), - ("https://imgur.com/zzzzzzz", { # not found + # not found + ("https://imgur.com/zzzzzzz", { "exception": exception.HttpError, }), ("https://m.imgur.com/r/Celebs/iHJ7tsM"), - ("https://www.imgur.com/21yMxCS"), # www - ("https://m.imgur.com/21yMxCS"), # mobile - ("https://imgur.com/zxaY6"), # 5 character key - ("https://imgur.io/zxaY6"), # .io - ("https://i.imgur.com/21yMxCS.png"), # direct link - ("https://i.imgur.io/21yMxCS.png"), # direct link .io - ("https://i.imgur.com/21yMxCSh.png"), # direct link thumbnail - ("https://i.imgur.com/zxaY6.gif"), # direct link (short) - ("https://i.imgur.com/zxaY6s.gif"), # direct link (short; thumb) + # www + ("https://www.imgur.com/21yMxCS"), + # mobile + ("https://m.imgur.com/21yMxCS"), + # 5 character key + ("https://imgur.com/zxaY6"), + # .io + ("https://imgur.io/zxaY6"), + # direct link + ("https://i.imgur.com/21yMxCS.png"), + # direct link .io + ("https://i.imgur.io/21yMxCS.png"), + # direct link thumbnail + ("https://i.imgur.com/21yMxCSh.png"), + # direct link (short) + ("https://i.imgur.com/zxaY6.gif"), + # direct link (short; thumb) + ("https://i.imgur.com/zxaY6s.gif"), ) def items(self): @@ -201,21 +214,27 @@ class ImgurAlbumExtractor(ImgurExtractor): "width" : int, }, }), - ("https://imgur.com/a/eD9CT", { # large album + # large album + ("https://imgur.com/a/eD9CT", { "url": "de748c181a04d18bef1de9d4f4866ef0a06d632b", }), - ("https://imgur.com/a/RhJXhVT/all", { # 7 character album hash + # 7 character album hash + ("https://imgur.com/a/RhJXhVT/all", { "url": "695ef0c950023362a0163ee5041796300db76674", }), ("https://imgur.com/a/TcBmQ", { "exception": exception.HttpError, }), - ("https://imgur.com/a/pjOnJA0", { # empty, no 'media' (#2557) + # empty, no 'media' (#2557) + ("https://imgur.com/a/pjOnJA0", { "count": 0, }), - ("https://www.imgur.com/a/TcBmP"), # www - ("https://imgur.io/a/TcBmP"), # .io - ("https://m.imgur.com/a/TcBmP"), # mobile + # www + ("https://www.imgur.com/a/TcBmP"), + # .io + ("https://imgur.io/a/TcBmP"), + # mobile + ("https://m.imgur.com/a/TcBmP"), ) def items(self): @@ -250,7 +269,8 @@ class ImgurGalleryExtractor(ImgurExtractor): subcategory = "gallery" pattern = BASE_PATTERN + r"/(?:gallery|t/\w+)/(\w{7}|\w{5})" test = ( - ("https://imgur.com/gallery/zf2fIms", { # non-album gallery (#380) + # non-album gallery (#380) + ("https://imgur.com/gallery/zf2fIms", { "pattern": "https://imgur.com/zf2fIms", }), ("https://imgur.com/gallery/eD9CT", { @@ -258,7 +278,8 @@ class ImgurGalleryExtractor(ImgurExtractor): }), ("https://imgur.com/t/unmuted/26sEhNr"), ("https://imgur.com/t/cat/qSB8NbN"), - ("https://imgur.io/t/cat/qSB8NbN"), # .io + # .io + ("https://imgur.io/t/cat/qSB8NbN"), ) def items(self): diff --git a/gallery_dl/extractor/keenspot.py b/gallery_dl/extractor/keenspot.py index b5d77380..78454a6f 100644 --- a/gallery_dl/extractor/keenspot.py +++ b/gallery_dl/extractor/keenspot.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,23 +21,28 @@ class KeenspotComicExtractor(Extractor): archive_fmt = "{comic}_{filename}" pattern = r"(?:https?://)?(?!www\.|forums\.)([\w-]+)\.keenspot\.com(/.+)?" test = ( - ("http://marksmen.keenspot.com/", { # link + # link + ("http://marksmen.keenspot.com/", { "range": "1-3", "url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", }), - ("http://barkercomic.keenspot.com/", { # id + # id + ("http://barkercomic.keenspot.com/", { "range": "1-3", "url": "c4080926db18d00bac641fdd708393b7d61379e6", }), - ("http://crowscare.keenspot.com/", { # id v2 + # id v2 + ("http://crowscare.keenspot.com/", { "range": "1-3", "url": "a00e66a133dd39005777317da90cef921466fcaa" }), - ("http://supernovas.keenspot.com/", { # ks + # ks + ("http://supernovas.keenspot.com/", { "range": "1-3", "url": "de21b12887ef31ff82edccbc09d112e3885c3aab" }), - ("http://twokinds.keenspot.com/comic/1066/", { # "random" access + # "random" access + ("http://twokinds.keenspot.com/comic/1066/", { "range": "1-3", "url": "6a784e11370abfb343dcad9adbb7718f9b7be350", }) diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index e12e56b4..e82ac91a 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -112,9 +112,9 @@ class MangadexChapterExtractor(MangadexExtractor): ("https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", { "exception": exception.StopExtraction, }), - # 'externalUrl', but still downloadable (#2503) + # 'externalUrl', but still downloadable / 404 (#2503) ("https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", { - "count": 0, # 404 + "count": 0, }), ) diff --git a/gallery_dl/extractor/pixnet.py b/gallery_dl/extractor/pixnet.py index 15be5635..7a5009cf 100644 --- a/gallery_dl/extractor/pixnet.py +++ b/gallery_dl/extractor/pixnet.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2020 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -11,7 +11,6 @@ from .common import Extractor, Message from .. import text, exception - BASE_PATTERN = r"(?:https?://)?(?!www\.)([\w-]+)\.pixnet.net" @@ -109,8 +108,9 @@ class PixnetSetExtractor(PixnetExtractor): "url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", "keyword": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", }), + # password-protected ("https://sky92100.pixnet.net/album/set/17492544", { - "count": 0, # password-protected + "count": 0, }), ) diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index ba571bbd..d10c412b 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -265,27 +265,32 @@ class ReactorPostExtractor(ReactorExtractor): "url": "dfc74d150d7267384d8c229c4b82aa210755daa0", }), ("http://anime.reactor.cc/post/3576250"), - ("http://joyreactor.com/post/3721876", { # single image + # single image + ("http://joyreactor.com/post/3721876", { "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" r"/cartoon-painting-monster-lake-4841316.jpeg", "count": 1, "keyword": "2207a7dfed55def2042b6c2554894c8d7fda386e", }), - ("http://joyreactor.com/post/3713804", { # 4 images + # 4 images + ("http://joyreactor.com/post/3713804", { "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" r"/movie-tv-godzilla-monsters-\d+\.jpeg", "count": 4, "keyword": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", }), - ("http://joyreactor.com/post/3726210", { # gif / video + # gif / video + ("http://joyreactor.com/post/3726210", { "url": "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", "keyword": "8949d9d5fc469dab264752432efbaa499561664a", }), - ("http://joyreactor.com/post/3668724", { # youtube embed + # youtube embed + ("http://joyreactor.com/post/3668724", { "url": "bf1666eddcff10c9b58f6be63fa94e4e13074214", "keyword": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", }), - ("http://joyreactor.cc/post/1299", { # "malformed" JSON + # "malformed" JSON + ("http://joyreactor.cc/post/1299", { "url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", }), ("http://pornreactor.cc/post/863166", { diff --git a/gallery_dl/extractor/tapas.py b/gallery_dl/extractor/tapas.py index 0e09e22a..53338e9a 100644 --- a/gallery_dl/extractor/tapas.py +++ b/gallery_dl/extractor/tapas.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -112,7 +112,8 @@ class TapasSeriesExtractor(TapasExtractor): "pattern": r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", "count": 132, }), - ("https://tapas.io/series/yona", { # mature + # mature + ("https://tapas.io/series/yona", { "count": 26, }), ) diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 9adc3ab1..e8c99294 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -287,23 +287,28 @@ class TumblrUserExtractor(TumblrExtractor): "count": 3, "options": (("posts", "all"), ("external", True)) }), - ("https://mikf123-hidden.tumblr.com/", { # dashboard-only + # dashboard-only + ("https://mikf123-hidden.tumblr.com/", { "options": (("access-token", None),), "exception": exception.AuthorizationError, }), - ("https://mikf123-hidden.tumblr.com/", { # dashboard-only + # dashboard-only + ("https://mikf123-hidden.tumblr.com/", { "count": 2, "keyword": {"tags": ["test", "hidden"]}, }), - ("https://mikf123-private.tumblr.com/", { # password protected + # password protected + ("https://mikf123-private.tumblr.com/", { "count": 2, "keyword": {"tags": ["test", "private"]}, }), - ("https://mikf123-private-hidden.tumblr.com/", { # both + # dashboard-only & password protected + ("https://mikf123-private-hidden.tumblr.com/", { "count": 2, "keyword": {"tags": ["test", "private", "hidden"]}, }), - ("https://mikf123.tumblr.com/", { # date-min/-max/-format (#337) + # date-min/-max/-format (#337) + ("https://mikf123.tumblr.com/", { "count": 4, "options": (("date-min", "201804"), ("date-max", "201805"), ("date-format", "%Y%m")) @@ -362,12 +367,14 @@ class TumblrPostExtractor(TumblrExtractor): ("https://mikf123.tumblr.com/post/181022380064/chat-post", { "count": 0, }), + # high-quality images (#1846) ("https://kichatundk.tumblr.com/post/654953419288821760", { - "count": 2, # high-quality images (#1846) + "count": 2, "content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", }), + # high-quality images (#1344) ("https://hameru-is-cool.tumblr.com/post/639261855227002880", { - "count": 2, # high-quality images (#1344) + "count": 2, "content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", }), ("https://mikf123.tumblr.com/image/689860196535762944", { @@ -376,8 +383,9 @@ class TumblrPostExtractor(TumblrExtractor): r"/8f1131551cef6e74-bc/s99999x99999" r"/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", }), + # HTML response (#297) ("http://ziemniax.tumblr.com/post/109697912859/", { - "exception": exception.NotFoundError, # HTML response (#297) + "exception": exception.NotFoundError, }), ("http://demo.tumblr.com/image/459265350"), ("https://www.tumblr.com/blog/view/smarties-art/686047436641353728"), From 8c0a656ccf8c4357de63e8f52ed9b616019dfa50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 6 Sep 2023 18:23:22 +0200 Subject: [PATCH 29/53] publish export_tests.py script --- scripts/export_tests.py | 299 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 scripts/export_tests.py diff --git a/scripts/export_tests.py b/scripts/export_tests.py new file mode 100644 index 00000000..e836eb9d --- /dev/null +++ b/scripts/export_tests.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2023 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import re +import sys +import itertools +import collections +from gallery_dl import extractor + + +FORMAT = '''\ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +{imports} + + +__tests__ = ( +{tests}\ +) +''' + + +def pprint(obj, indent=0, lmin=9, lmax=16): + + if isinstance(obj, str): + if obj.startswith("lit:"): + return f'''{obj[4:]}''' + prefix = "r" if "\\" in obj or obj.startswith("re:") else "" + quote = '"""' if "\n" in obj else '"' + return f'''{prefix}{quote}{obj}{quote}''' + + if isinstance(obj, bytes): + return f'''b"{str(obj)[2:-1]}"''' + + if isinstance(obj, type): + if obj.__module__ == "builtins": + return f'''{obj.__name__}''' + + name = obj.__module__.rpartition(".")[2] + if name[0].isdecimal(): + name = f"_{name}" + return f'''{name}.{obj.__name__}''' + + if isinstance(obj, dict): + if not obj: + return "{}" + + if len(obj) >= 2 or not indent: + ws = " " * indent + + lkey = max(map(len, obj)) + if not indent: + if lkey < lmin: + lkey = lmin + elif lkey > lmax: + lkey = lmax + + lines = [] + lines.append("{") + for key, value in obj.items(): + if key.startswith("#blank-"): + lines.append("") + else: + lines.append( + f'''{ws} "{key}"''' + f'''{' '*(lkey - len(key))}: ''' + f'''{pprint(value, indent+4)},''' + ) + lines.append(f'''{ws}}}''') + return "\n".join(lines) + else: + key, value = obj.popitem() + return f'''{{"{key}": {pprint(value)}}}''' + + if isinstance(obj, list): + if not obj: + return "[]" + + if len(obj) >= 2: + ws = " " * indent + + lines = [] + lines.append("[") + lines.extend( + f'''{ws} {pprint(value, indent+4)},''' + for value in obj + ) + lines.append(f'''{ws}]''') + return "\n".join(lines) + else: + return f'''[{pprint(obj[0])}]''' + + if isinstance(obj, tuple): + if len(obj) == 1: + return f'''({pprint(obj[0], indent+4)},)''' + return f'''({", ".join(pprint(v, indent+4) for v in obj)})''' + + else: + return f'''{obj}''' + + +def extract_tests_from_source(lines): + tests = {} + + match_url = re.compile( + r''' (?:test = | )?\(\(?"([^"]+)"(.*)''').match + match_end = re.compile( + r" (\}\)| \}\),)\n$").match + first = 0 + url = "" + + for index, line in enumerate(lines): + if first and match_end(line): + tests[url] = lines[first-1:index+1] + first = 0 + + elif (m := match_url(line)): + offset = index + while not m[2]: + offset += 1 + next = lines[offset] + line = line[:-2] + next[next.index('"')+1:] + m = match_url(line) + url = m[1] + if m[2] in (",)", "),"): + tests[url] = lines[index-1:index+1] + first = 0 + else: + first = index + + return tests + + +def get_test_source(extr, *, cache={}): + try: + tests = cache[extr.__module__] + except KeyError: + path = sys.modules[extr.__module__].__file__ + with open(path) as fp: + lines = fp.readlines() + tests = cache[extr.__module__] = extract_tests_from_source(lines) + return tests.get(extr.url) or ("",) + return tests[extr.url] + + +def comment_from_source(source): + match = re.match(r"\s+#\s*(.+)", source[0]) + return match[1] if match else "" + + +def build_test(extr, data): + source = get_test_source(extr) + comment = comment_from_source(source) + + head = { + "#url" : extr.url, + "#comment" : comment.replace('"', "'"), + "#category": (extr.basecategory, + extr.category, + extr.subcategory), + "#class" : extr.__class__, + } + + if not comment: + del head["#comment"] + + instr = {} + + if not data: + data = {} + if (options := data.pop("options", None)): + instr["#options"] = { + name: value + for name, value in options + } + if (pattern := data.pop("pattern", None)): + if pattern in PATTERNS: + cls = PATTERNS[pattern] + pattern = f"lit:{pprint(cls)}.pattern" + instr["#pattern"] = pattern + if (exception := data.pop("exception", None)): + instr["#exception"] = exception + if (range := data.pop("range", None)): + instr["#range"] = range + if (count := data.pop("count", None)) is not None: + instr["#count"] = count + if (archive := data.pop("archive", None)) is not None: + instr["#archive"] = archive + if (extractor := data.pop("extractor", None)) is not None: + instr["#extractor"] = extractor + if (url := data.pop("url", None)): + instr["#sha1_url"] = url + if (metadata := data.pop("keyword", None)): + if isinstance(metadata, str) and len(metadata) == 40: + instr["#sha1_metadata"] = metadata + metadata = {} + if (content := data.pop("content", None)): + instr["#sha1_content"] = content + + if data: + print(extr) + for k in data: + print(k) + exit() + + return head, instr, metadata + + +def collect_patterns(): + return { + cls.pattern.pattern: cls + for cls in extractor._list_classes() + } + + +def collect_tests(): + tests = collections.defaultdict(list) + + for cls in extractor._list_classes(): + for url, data in cls._get_tests(): + + extr = cls.from_url(url) + test = build_test(extr, data) + tests[extr.category].append(test) + + return tests + + +def export_tests(category, data): + imports = {} + tests = [] + + for head, instr, metadata in data: + + for v in itertools.chain( + head.values(), + instr.values() if instr else (), + metadata.values() if metadata else (), + ): + if not isinstance(v, type) or v.__module__ == "builtins": + continue + + module, _, name = v.__module__.rpartition(".") + if name[0].isdecimal(): + stmt = f'''\ +{module.partition(".")[0]} = __import__("{v.__module__}") +_{name} = getattr({module}, "{name}")''' + elif module: + stmt = f"from {module} import {name}" + else: + stmt = f"import {name}" + imports[v.__module__] = stmt + + test = pprint(head) + if instr: + test = f"{test[:-2]}{pprint(instr)[1:]}" + if metadata: + for k, v in metadata.items(): + if v == "type:datetime": + imports["datetime"] = "import datetime" + metadata[k] = "lit:datetime.datetime" + test = f"{test[:-1]}{pprint(metadata, lmin=0)[1:]}" + + tests.append(f"{test},\n\n") + + return FORMAT.format( + imports="\n".join(imports.values()), + tests="".join(tests), + ) + + +PATTERNS = None +DIRECTORY = "/tmp/_/results" + + +def main(): + os.makedirs(DIRECTORY, exist_ok=True) + + global PATTERNS + PATTERNS = collect_patterns() + + for category, tests in collect_tests().items(): + with open(f"{DIRECTORY}/{category}.py", "w") as fp: + fp.write(export_tests(category, tests)) + + +if __name__ == "__main__": + main() From c2977451189b32098d992377bcdbc324953d929d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 9 Sep 2023 19:13:56 +0200 Subject: [PATCH 30/53] add -t and -c command-line arguments to export_tests.py --- scripts/export_tests.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/scripts/export_tests.py b/scripts/export_tests.py index e836eb9d..4752cae8 100644 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -206,6 +206,8 @@ def build_test(extr, data): instr["#sha1_metadata"] = metadata metadata = {} if (content := data.pop("content", None)): + if isinstance(content, tuple): + content = list(content) instr["#sha1_content"] = content if data: @@ -224,20 +226,22 @@ def collect_patterns(): } -def collect_tests(): +def collect_tests(whitelist=None): tests = collections.defaultdict(list) for cls in extractor._list_classes(): for url, data in cls._get_tests(): extr = cls.from_url(url) + if whitelist and extr.category not in whitelist: + continue test = build_test(extr, data) - tests[extr.category].append(test) + tests[f"{extr.category}_{extr.subcategory}"].append(test) return tests -def export_tests(category, data): +def export_tests(data): imports = {} tests = [] @@ -285,14 +289,33 @@ DIRECTORY = "/tmp/_/results" def main(): - os.makedirs(DIRECTORY, exist_ok=True) + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", "--target", + help="target directory", + ) + parser.add_argument( + "-c", "--category", action="append", + help="extractor categories to export", + ) + + args = parser.parse_args() + + if not args.target: + args.target = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "test", "results", + ) global PATTERNS PATTERNS = collect_patterns() - for category, tests in collect_tests().items(): - with open(f"{DIRECTORY}/{category}.py", "w") as fp: - fp.write(export_tests(category, tests)) + os.makedirs(args.target, exist_ok=True) + for name, tests in collect_tests(args.category).items(): + with open(f"{args.target}/{name}.py", "w") as fp: + fp.write(export_tests(tests)) if __name__ == "__main__": From 28b79354fb4f4ddd1746e52c72bd46b0a7691023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 9 Sep 2023 22:34:03 +0200 Subject: [PATCH 31/53] fix for categories containing '.' Files with ':' in their name cannot be imported, as __import__() will try to interpret them as submodules. --- scripts/export_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/export_tests.py diff --git a/scripts/export_tests.py b/scripts/export_tests.py old mode 100644 new mode 100755 index 4752cae8..17a684b3 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -12,6 +12,8 @@ import re import sys import itertools import collections + +import util from gallery_dl import extractor @@ -314,8 +316,9 @@ def main(): os.makedirs(args.target, exist_ok=True) for name, tests in collect_tests(args.category).items(): - with open(f"{args.target}/{name}.py", "w") as fp: - fp.write(export_tests(tests)) + name = name.replace(".", "") + with util.lazy(f"{args.target}/{name}.py") as file: + file.write(export_tests(tests)) if __name__ == "__main__": From b611bf9629d42ef428a9bc3d060553240aa8e96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sat, 9 Sep 2023 22:36:23 +0200 Subject: [PATCH 32/53] fix '"' escapes --- scripts/export_tests.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/export_tests.py b/scripts/export_tests.py index 17a684b3..1a09e107 100755 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -38,8 +38,20 @@ def pprint(obj, indent=0, lmin=9, lmax=16): if isinstance(obj, str): if obj.startswith("lit:"): return f'''{obj[4:]}''' - prefix = "r" if "\\" in obj or obj.startswith("re:") else "" - quote = '"""' if "\n" in obj else '"' + + if "\\" in obj or obj.startswith("re:"): + prefix = "r" + else: + prefix = "" + + if "\n" in obj: + quote = '"""' + elif '"' in obj: + obj = re.sub(r'(?<!\\)"', '\\"', obj) + quote = '"' + else: + quote = '"' + return f'''{prefix}{quote}{obj}{quote}''' if isinstance(obj, bytes): From a833c244c8991b72061734135e6d7cb6c7166a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sun, 10 Sep 2023 14:45:01 +0200 Subject: [PATCH 33/53] add exported extractor results --- scripts/export_tests.py | 2 +- setup.cfg | 2 +- test/results/2chan.py | 34 ++ test/results/2chen.py | 68 +++ test/results/35photo.py | 75 +++ test/results/3dbooru.py | 50 ++ test/results/4chan.py | 37 ++ test/results/4chanarchives.py | 49 ++ test/results/4plebs.py | 36 ++ test/results/500px.py | 103 ++++ test/results/8chan.py | 91 ++++ test/results/8kun.py | 39 ++ test/results/8muses.py | 73 +++ test/results/94chan.py | 45 ++ test/results/__init__.py | 32 ++ test/results/acidimg.py | 20 + test/results/adultempire.py | 28 + test/results/aibooru.py | 44 ++ test/results/allgirlbooru.py | 47 ++ test/results/animereactor.py | 29 ++ test/results/architizer.py | 40 ++ test/results/archivedmoe.py | 44 ++ test/results/archiveofsins.py | 43 ++ test/results/artstation.py | 160 ++++++ test/results/aryion.py | 86 ++++ test/results/atfbooru.py | 39 ++ test/results/b4k.py | 30 ++ test/results/baraag.py | 36 ++ test/results/bbc.py | 49 ++ test/results/bbw-chan.py | 39 ++ test/results/bcbnsfw.py | 25 + test/results/behance.py | 88 ++++ test/results/bitly.py | 19 + test/results/blogger.py | 107 ++++ test/results/booruvar.py | 40 ++ test/results/bunkr.py | 100 ++++ test/results/catbox.py | 59 +++ test/results/cavemanon.py | 44 ++ test/results/chelseacrew.py | 23 + test/results/comicvine.py | 27 + test/results/coomerparty.py | 19 + test/results/cyberdrop.py | 41 ++ test/results/danbooru.py | 103 ++++ test/results/derpibooru.py | 116 +++++ test/results/desktopography.py | 29 ++ test/results/desuarchive.py | 36 ++ test/results/deviantart.py | 732 +++++++++++++++++++++++++++ test/results/directlink.py | 62 +++ test/results/drawfriends.py | 29 ++ test/results/dynastyscans.py | 59 +++ test/results/e621.py | 128 +++++ test/results/e6ai.py | 68 +++ test/results/e926.py | 87 ++++ test/results/endchan.py | 39 ++ test/results/erome.py | 53 ++ test/results/exhentai.py | 132 +++++ test/results/fallenangels.py | 51 ++ test/results/fanbox.py | 104 ++++ test/results/fanleaks.py | 67 +++ test/results/fantia.py | 67 +++ test/results/fapachi.py | 44 ++ test/results/fapello.py | 100 ++++ test/results/fappic.py | 26 + test/results/fashionnova.py | 45 ++ test/results/fireden.py | 36 ++ test/results/flickr.py | 162 ++++++ test/results/foalcon.py | 29 ++ test/results/furaffinity.py | 161 ++++++ test/results/furbooru.py | 33 ++ test/results/fuskator.py | 47 ++ test/results/gelbooru.py | 142 ++++++ test/results/generic.py | 68 +++ test/results/giantessbooru.py | 50 ++ test/results/gofile.py | 57 +++ test/results/hbrowse.py | 28 + test/results/hentai2read.py | 74 +++ test/results/hentaicosplays.py | 45 ++ test/results/hentaifoundry.py | 152 ++++++ test/results/hentaifox.py | 108 ++++ test/results/hentaihand.py | 57 +++ test/results/hentaihere.py | 65 +++ test/results/hiperdex.py | 136 +++++ test/results/hitomi.py | 139 +++++ test/results/horne.py | 96 ++++ test/results/hotleak.py | 104 ++++ test/results/hypnohub.py | 77 +++ test/results/idolcomplex.py | 55 ++ test/results/illusioncardsbooru.py | 34 ++ test/results/imagebam.py | 78 +++ test/results/imagechest.py | 44 ++ test/results/imagefap.py | 191 +++++++ test/results/imagetwist.py | 44 ++ test/results/imagevenue.py | 34 ++ test/results/imgbb.py | 72 +++ test/results/imgbox.py | 52 ++ test/results/imgclick.py | 20 + test/results/imgspice.py | 20 + test/results/imgth.py | 34 ++ test/results/imgur.py | 367 ++++++++++++++ test/results/imxto.py | 65 +++ test/results/inkbunny.py | 146 ++++++ test/results/instagram.py | 262 ++++++++++ test/results/issuu.py | 47 ++ test/results/itaku.py | 82 +++ test/results/itchio.py | 33 ++ test/results/joyreactor.py | 108 ++++ test/results/jpgfish.py | 149 ++++++ test/results/kabeuchi.py | 27 + test/results/keenspot.py | 55 ++ test/results/kemonoparty.py | 232 +++++++++ test/results/khinsider.py | 30 ++ test/results/kohlchan.py | 39 ++ test/results/komikcast.py | 41 ++ test/results/konachan.py | 82 +++ test/results/lensdump.py | 48 ++ test/results/lesbianenergy.py | 45 ++ test/results/lexica.py | 42 ++ test/results/lightroom.py | 31 ++ test/results/livedoor.py | 66 +++ test/results/lolibooru.py | 45 ++ test/results/loudbooru.py | 37 ++ test/results/loungeunderwear.py | 23 + test/results/luscious.py | 115 +++++ test/results/mangadex.py | 116 +++++ test/results/mangafox.py | 71 +++ test/results/mangahere.py | 77 +++ test/results/mangakakalot.py | 40 ++ test/results/mangalife.py | 69 +++ test/results/manganelo.py | 91 ++++ test/results/mangapark.py | 140 +++++ test/results/mangaread.py | 122 +++++ test/results/mangasee.py | 69 +++ test/results/mangoxo.py | 42 ++ test/results/mastodonsocial.py | 89 ++++ test/results/mememuseum.py | 38 ++ test/results/michaelscameras.py | 23 + test/results/misskeyio.py | 62 +++ test/results/modcloth.py | 23 + test/results/myhentaigallery.py | 32 ++ test/results/myportfolio.py | 51 ++ test/results/naver.py | 45 ++ test/results/naverwebtoon.py | 96 ++++ test/results/newgrounds.py | 303 +++++++++++ test/results/nhentai.py | 97 ++++ test/results/nijie.py | 114 +++++ test/results/nitter1d4us.py | 62 +++ test/results/nitterit.py | 18 + test/results/nitterkavinrocks.py | 46 ++ test/results/nitterlacontrevoiefr.py | 73 +++ test/results/nitternet.py | 107 ++++ test/results/nitterunixfoxeu.py | 53 ++ test/results/nozomi.py | 106 ++++ test/results/nsfwalbum.py | 20 + test/results/nudecollect.py | 56 ++ test/results/ohpolly.py | 23 + test/results/omgmiamiswimwear.py | 25 + test/results/paheal.py | 97 ++++ test/results/palanq.py | 36 ++ test/results/patreon.py | 101 ++++ test/results/pawoo.py | 38 ++ test/results/photobucket.py | 68 +++ test/results/photovogue.py | 50 ++ test/results/picarto.py | 22 + test/results/piczel.py | 57 +++ test/results/pillowfort.py | 94 ++++ test/results/pinterest.py | 155 ++++++ test/results/pinupgirlclothing.py | 23 + test/results/pixhost.py | 28 + test/results/pixiv.py | 520 +++++++++++++++++++ test/results/pixnet.py | 86 ++++ test/results/plurk.py | 35 ++ test/results/poipiku.py | 76 +++ test/results/ponybooru.py | 39 ++ test/results/pornhub.py | 118 +++++ test/results/pornpics.py | 105 ++++ test/results/pornreactor.py | 66 +++ test/results/postimg.py | 46 ++ test/results/powermanga.py | 36 ++ test/results/pururin.py | 56 ++ test/results/raidlondon.py | 23 + test/results/rbt.py | 43 ++ test/results/reactor.py | 36 ++ test/results/readcomiconline.py | 36 ++ test/results/realbooru.py | 44 ++ test/results/recursive.py | 18 + test/results/reddit.py | 243 +++++++++ test/results/redgifs.py | 152 ++++++ test/results/rule34.py | 76 +++ test/results/rule34us.py | 47 ++ test/results/safebooru.py | 47 ++ test/results/sakugabooru.py | 35 ++ test/results/sankaku.py | 194 +++++++ test/results/sankakucomplex.py | 69 +++ test/results/seiga.py | 104 ++++ test/results/senmanga.py | 65 +++ test/results/sexcom.py | 103 ++++ test/results/simplyhentai.py | 75 +++ test/results/skeb.py | 80 +++ test/results/slickpic.py | 48 ++ test/results/slideshare.py | 47 ++ test/results/smugloli.py | 49 ++ test/results/smugmug.py | 97 ++++ test/results/soundgasm.py | 47 ++ test/results/speakerdeck.py | 20 + test/results/subscribestar.py | 82 +++ test/results/sushiski.py | 37 ++ test/results/tapas.py | 76 +++ test/results/tbib.py | 33 ++ test/results/tcbscans.py | 69 +++ test/results/tco.py | 27 + test/results/telegraph.py | 84 +++ test/results/tentaclerape.py | 47 ++ test/results/test.py | 29 ++ test/results/thebarchive.py | 36 ++ test/results/thecollection.py | 34 ++ test/results/toyhouse.py | 75 +++ test/results/tsumino.py | 72 +++ test/results/tumblr.py | 344 +++++++++++++ test/results/tumblrgallery.py | 33 ++ test/results/turboimagehost.py | 20 + test/results/twibooru.py | 85 ++++ test/results/twitter.py | 574 +++++++++++++++++++++ test/results/unique-vintage.py | 23 + test/results/unsplash.py | 124 +++++ test/results/uploadir.py | 62 +++ test/results/vanillarock.py | 35 ++ test/results/vidyart2.py | 29 ++ test/results/vipergirls.py | 47 ++ test/results/vipr.py | 19 + test/results/vk.py | 100 ++++ test/results/vsco.py | 70 +++ test/results/wallhaven.py | 104 ++++ test/results/wallpapercave.py | 18 + test/results/warosu.py | 28 + test/results/weasyl.py | 95 ++++ test/results/webmshare.py | 55 ++ test/results/webtoons.py | 73 +++ test/results/weibo.py | 227 +++++++++ test/results/wikiart.py | 50 ++ test/results/wikieat.py | 39 ++ test/results/wikifeet.py | 51 ++ test/results/wikifeetx.py | 31 ++ test/results/windsorstore.py | 23 + test/results/xbooru.py | 41 ++ test/results/xbunkr.py | 28 + test/results/xhamster.py | 114 +++++ test/results/xvideos.py | 58 +++ test/results/yandere.py | 78 +++ test/results/ytdl.py | 17 + test/results/zerochan.py | 67 +++ 250 files changed, 18610 insertions(+), 2 deletions(-) create mode 100644 test/results/2chan.py create mode 100644 test/results/2chen.py create mode 100644 test/results/35photo.py create mode 100644 test/results/3dbooru.py create mode 100644 test/results/4chan.py create mode 100644 test/results/4chanarchives.py create mode 100644 test/results/4plebs.py create mode 100644 test/results/500px.py create mode 100644 test/results/8chan.py create mode 100644 test/results/8kun.py create mode 100644 test/results/8muses.py create mode 100644 test/results/94chan.py create mode 100644 test/results/__init__.py create mode 100644 test/results/acidimg.py create mode 100644 test/results/adultempire.py create mode 100644 test/results/aibooru.py create mode 100644 test/results/allgirlbooru.py create mode 100644 test/results/animereactor.py create mode 100644 test/results/architizer.py create mode 100644 test/results/archivedmoe.py create mode 100644 test/results/archiveofsins.py create mode 100644 test/results/artstation.py create mode 100644 test/results/aryion.py create mode 100644 test/results/atfbooru.py create mode 100644 test/results/b4k.py create mode 100644 test/results/baraag.py create mode 100644 test/results/bbc.py create mode 100644 test/results/bbw-chan.py create mode 100644 test/results/bcbnsfw.py create mode 100644 test/results/behance.py create mode 100644 test/results/bitly.py create mode 100644 test/results/blogger.py create mode 100644 test/results/booruvar.py create mode 100644 test/results/bunkr.py create mode 100644 test/results/catbox.py create mode 100644 test/results/cavemanon.py create mode 100644 test/results/chelseacrew.py create mode 100644 test/results/comicvine.py create mode 100644 test/results/coomerparty.py create mode 100644 test/results/cyberdrop.py create mode 100644 test/results/danbooru.py create mode 100644 test/results/derpibooru.py create mode 100644 test/results/desktopography.py create mode 100644 test/results/desuarchive.py create mode 100644 test/results/deviantart.py create mode 100644 test/results/directlink.py create mode 100644 test/results/drawfriends.py create mode 100644 test/results/dynastyscans.py create mode 100644 test/results/e621.py create mode 100644 test/results/e6ai.py create mode 100644 test/results/e926.py create mode 100644 test/results/endchan.py create mode 100644 test/results/erome.py create mode 100644 test/results/exhentai.py create mode 100644 test/results/fallenangels.py create mode 100644 test/results/fanbox.py create mode 100644 test/results/fanleaks.py create mode 100644 test/results/fantia.py create mode 100644 test/results/fapachi.py create mode 100644 test/results/fapello.py create mode 100644 test/results/fappic.py create mode 100644 test/results/fashionnova.py create mode 100644 test/results/fireden.py create mode 100644 test/results/flickr.py create mode 100644 test/results/foalcon.py create mode 100644 test/results/furaffinity.py create mode 100644 test/results/furbooru.py create mode 100644 test/results/fuskator.py create mode 100644 test/results/gelbooru.py create mode 100644 test/results/generic.py create mode 100644 test/results/giantessbooru.py create mode 100644 test/results/gofile.py create mode 100644 test/results/hbrowse.py create mode 100644 test/results/hentai2read.py create mode 100644 test/results/hentaicosplays.py create mode 100644 test/results/hentaifoundry.py create mode 100644 test/results/hentaifox.py create mode 100644 test/results/hentaihand.py create mode 100644 test/results/hentaihere.py create mode 100644 test/results/hiperdex.py create mode 100644 test/results/hitomi.py create mode 100644 test/results/horne.py create mode 100644 test/results/hotleak.py create mode 100644 test/results/hypnohub.py create mode 100644 test/results/idolcomplex.py create mode 100644 test/results/illusioncardsbooru.py create mode 100644 test/results/imagebam.py create mode 100644 test/results/imagechest.py create mode 100644 test/results/imagefap.py create mode 100644 test/results/imagetwist.py create mode 100644 test/results/imagevenue.py create mode 100644 test/results/imgbb.py create mode 100644 test/results/imgbox.py create mode 100644 test/results/imgclick.py create mode 100644 test/results/imgspice.py create mode 100644 test/results/imgth.py create mode 100644 test/results/imgur.py create mode 100644 test/results/imxto.py create mode 100644 test/results/inkbunny.py create mode 100644 test/results/instagram.py create mode 100644 test/results/issuu.py create mode 100644 test/results/itaku.py create mode 100644 test/results/itchio.py create mode 100644 test/results/joyreactor.py create mode 100644 test/results/jpgfish.py create mode 100644 test/results/kabeuchi.py create mode 100644 test/results/keenspot.py create mode 100644 test/results/kemonoparty.py create mode 100644 test/results/khinsider.py create mode 100644 test/results/kohlchan.py create mode 100644 test/results/komikcast.py create mode 100644 test/results/konachan.py create mode 100644 test/results/lensdump.py create mode 100644 test/results/lesbianenergy.py create mode 100644 test/results/lexica.py create mode 100644 test/results/lightroom.py create mode 100644 test/results/livedoor.py create mode 100644 test/results/lolibooru.py create mode 100644 test/results/loudbooru.py create mode 100644 test/results/loungeunderwear.py create mode 100644 test/results/luscious.py create mode 100644 test/results/mangadex.py create mode 100644 test/results/mangafox.py create mode 100644 test/results/mangahere.py create mode 100644 test/results/mangakakalot.py create mode 100644 test/results/mangalife.py create mode 100644 test/results/manganelo.py create mode 100644 test/results/mangapark.py create mode 100644 test/results/mangaread.py create mode 100644 test/results/mangasee.py create mode 100644 test/results/mangoxo.py create mode 100644 test/results/mastodonsocial.py create mode 100644 test/results/mememuseum.py create mode 100644 test/results/michaelscameras.py create mode 100644 test/results/misskeyio.py create mode 100644 test/results/modcloth.py create mode 100644 test/results/myhentaigallery.py create mode 100644 test/results/myportfolio.py create mode 100644 test/results/naver.py create mode 100644 test/results/naverwebtoon.py create mode 100644 test/results/newgrounds.py create mode 100644 test/results/nhentai.py create mode 100644 test/results/nijie.py create mode 100644 test/results/nitter1d4us.py create mode 100644 test/results/nitterit.py create mode 100644 test/results/nitterkavinrocks.py create mode 100644 test/results/nitterlacontrevoiefr.py create mode 100644 test/results/nitternet.py create mode 100644 test/results/nitterunixfoxeu.py create mode 100644 test/results/nozomi.py create mode 100644 test/results/nsfwalbum.py create mode 100644 test/results/nudecollect.py create mode 100644 test/results/ohpolly.py create mode 100644 test/results/omgmiamiswimwear.py create mode 100644 test/results/paheal.py create mode 100644 test/results/palanq.py create mode 100644 test/results/patreon.py create mode 100644 test/results/pawoo.py create mode 100644 test/results/photobucket.py create mode 100644 test/results/photovogue.py create mode 100644 test/results/picarto.py create mode 100644 test/results/piczel.py create mode 100644 test/results/pillowfort.py create mode 100644 test/results/pinterest.py create mode 100644 test/results/pinupgirlclothing.py create mode 100644 test/results/pixhost.py create mode 100644 test/results/pixiv.py create mode 100644 test/results/pixnet.py create mode 100644 test/results/plurk.py create mode 100644 test/results/poipiku.py create mode 100644 test/results/ponybooru.py create mode 100644 test/results/pornhub.py create mode 100644 test/results/pornpics.py create mode 100644 test/results/pornreactor.py create mode 100644 test/results/postimg.py create mode 100644 test/results/powermanga.py create mode 100644 test/results/pururin.py create mode 100644 test/results/raidlondon.py create mode 100644 test/results/rbt.py create mode 100644 test/results/reactor.py create mode 100644 test/results/readcomiconline.py create mode 100644 test/results/realbooru.py create mode 100644 test/results/recursive.py create mode 100644 test/results/reddit.py create mode 100644 test/results/redgifs.py create mode 100644 test/results/rule34.py create mode 100644 test/results/rule34us.py create mode 100644 test/results/safebooru.py create mode 100644 test/results/sakugabooru.py create mode 100644 test/results/sankaku.py create mode 100644 test/results/sankakucomplex.py create mode 100644 test/results/seiga.py create mode 100644 test/results/senmanga.py create mode 100644 test/results/sexcom.py create mode 100644 test/results/simplyhentai.py create mode 100644 test/results/skeb.py create mode 100644 test/results/slickpic.py create mode 100644 test/results/slideshare.py create mode 100644 test/results/smugloli.py create mode 100644 test/results/smugmug.py create mode 100644 test/results/soundgasm.py create mode 100644 test/results/speakerdeck.py create mode 100644 test/results/subscribestar.py create mode 100644 test/results/sushiski.py create mode 100644 test/results/tapas.py create mode 100644 test/results/tbib.py create mode 100644 test/results/tcbscans.py create mode 100644 test/results/tco.py create mode 100644 test/results/telegraph.py create mode 100644 test/results/tentaclerape.py create mode 100644 test/results/test.py create mode 100644 test/results/thebarchive.py create mode 100644 test/results/thecollection.py create mode 100644 test/results/toyhouse.py create mode 100644 test/results/tsumino.py create mode 100644 test/results/tumblr.py create mode 100644 test/results/tumblrgallery.py create mode 100644 test/results/turboimagehost.py create mode 100644 test/results/twibooru.py create mode 100644 test/results/twitter.py create mode 100644 test/results/unique-vintage.py create mode 100644 test/results/unsplash.py create mode 100644 test/results/uploadir.py create mode 100644 test/results/vanillarock.py create mode 100644 test/results/vidyart2.py create mode 100644 test/results/vipergirls.py create mode 100644 test/results/vipr.py create mode 100644 test/results/vk.py create mode 100644 test/results/vsco.py create mode 100644 test/results/wallhaven.py create mode 100644 test/results/wallpapercave.py create mode 100644 test/results/warosu.py create mode 100644 test/results/weasyl.py create mode 100644 test/results/webmshare.py create mode 100644 test/results/webtoons.py create mode 100644 test/results/weibo.py create mode 100644 test/results/wikiart.py create mode 100644 test/results/wikieat.py create mode 100644 test/results/wikifeet.py create mode 100644 test/results/wikifeetx.py create mode 100644 test/results/windsorstore.py create mode 100644 test/results/xbooru.py create mode 100644 test/results/xbunkr.py create mode 100644 test/results/xhamster.py create mode 100644 test/results/xvideos.py create mode 100644 test/results/yandere.py create mode 100644 test/results/ytdl.py create mode 100644 test/results/zerochan.py diff --git a/scripts/export_tests.py b/scripts/export_tests.py index 1a09e107..e95b0c2f 100755 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -250,7 +250,7 @@ def collect_tests(whitelist=None): if whitelist and extr.category not in whitelist: continue test = build_test(extr, data) - tests[f"{extr.category}_{extr.subcategory}"].append(test) + tests[extr.category].append(test) return tests diff --git a/setup.cfg b/setup.cfg index 56d71087..a66d7cf3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -exclude = build,archive +exclude = .git,__pycache__,build,dist,archive,test/results ignore = E203,E226,W504 per-file-ignores = setup.py: E501 diff --git a/test/results/2chan.py b/test/results/2chan.py new file mode 100644 index 00000000..0a1cdd69 --- /dev/null +++ b/test/results/2chan.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.2chan") +_2chan = getattr(gallery_dl.extractor, "2chan") + + +__tests__ = ( +{ + "#url" : "https://dec.2chan.net/70/res/14565.htm", + "#category": ("", "2chan", "thread"), + "#class" : _2chan._2chanThreadExtractor, + "#pattern" : r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", + "#count" : ">= 3", + + "board" : "70", + "board_name": "新板提案", + "com" : str, + "fsize" : r"re:\d+", + "name" : "名無し", + "no" : r"re:1[45]\d\d\d", + "now" : r"re:22/../..\(.\)..:..:..", + "post" : "無題", + "server" : "dec", + "thread" : "14565", + "tim" : r"re:^\d{13}$", + "time" : r"re:^\d{10}$", + "title" : "ﾋﾛｱｶ板", +}, + +) diff --git a/test/results/2chen.py b/test/results/2chen.py new file mode 100644 index 00000000..589053fa --- /dev/null +++ b/test/results/2chen.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.2chen") +_2chen = getattr(gallery_dl.extractor, "2chen") +import datetime + + +__tests__ = ( +{ + "#url" : "https://sturdychan.help/tv/268929", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, + "#pattern" : r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", + "#count" : ">= 179", + + "board" : "tv", + "date" : datetime.datetime, + "hash" : r"re:[0-9a-f]{40}", + "name" : "Anonymous", + "no" : r"re:\d+", + "thread": "268929", + "time" : int, + "title" : "「/ttg/ #118: 🇧🇷 edition」", + "url" : str, +}, + +{ + "#url" : "https://2chen.club/tv/1", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, +}, + +{ + "#url" : "https://2chen.moe/jp/303786", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, +}, + +{ + "#url" : "https://sturdychan.help/co/", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, + "#pattern" : _2chen._2chenThreadExtractor.pattern, +}, + +{ + "#url" : "https://2chen.moe/co", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +{ + "#url" : "https://2chen.club/tv", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +{ + "#url" : "https://2chen.moe/co/catalog", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +) diff --git a/test/results/35photo.py b/test/results/35photo.py new file mode 100644 index 00000000..b9c7673a --- /dev/null +++ b/test/results/35photo.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.35photo") +_35photo = getattr(gallery_dl.extractor, "35photo") + + +__tests__ = ( +{ + "#url" : "https://35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, + "#pattern" : r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", + "#count" : 9, +}, + +{ + "#url" : "https://35photo.pro/suhoveev", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, + "#count" : ">= 33", +}, + +{ + "#url" : "https://en.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, +}, + +{ + "#url" : "https://ru.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, +}, + +{ + "#url" : "https://35photo.pro/tags/landscape/", + "#category": ("", "35photo", "tag"), + "#class" : _35photo._35photoTagExtractor, + "#range" : "1-25", + "#count" : 25, + "#archive" : False, +}, + +{ + "#url" : "https://35photo.pro/genre_109/", + "#category": ("", "35photo", "genre"), + "#class" : _35photo._35photoGenreExtractor, +}, + +{ + "#url" : "https://35photo.pro/photo_753340/", + "#category": ("", "35photo", "image"), + "#class" : _35photo._35photoImageExtractor, + "#count" : 1, + + "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", + "id" : 753340, + "title" : "Winter walk", + "description": str, + "tags" : list, + "views" : int, + "favorites" : int, + "score" : int, + "type" : 0, + "date" : "15 авг, 2014", + "user" : "liya", + "user_id" : 20415, + "user_name" : "Liya Mirzaeva", +}, + +) diff --git a/test/results/3dbooru.py b/test/results/3dbooru.py new file mode 100644 index 00000000..d3d407bd --- /dev/null +++ b/test/results/3dbooru.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.3dbooru") +_3dbooru = getattr(gallery_dl.extractor, "3dbooru") + + +__tests__ = ( +{ + "#url" : "http://behoimi.org/post?tags=himekawa_azuru+dress", + "#category": ("booru", "3dbooru", "tag"), + "#class" : _3dbooru._3dbooruTagExtractor, + "#sha1_url" : "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", + "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", +}, + +{ + "#url" : "http://behoimi.org/pool/show/27", + "#category": ("booru", "3dbooru", "pool"), + "#class" : _3dbooru._3dbooruPoolExtractor, + "#sha1_url" : "da75d2d1475449d5ef0c266cb612683b110a30f2", + "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", +}, + +{ + "#url" : "http://behoimi.org/post/show/140852", + "#category": ("booru", "3dbooru", "post"), + "#class" : _3dbooru._3dbooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_url" : "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", + "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", + + "tags_character": "furude_rika", + "tags_copyright": "higurashi_no_naku_koro_ni", + "tags_model" : "himekawa_azuru", + "tags_general" : str, +}, + +{ + "#url" : "http://behoimi.org/post/popular_by_month?month=2&year=2013", + "#category": ("booru", "3dbooru", "popular"), + "#class" : _3dbooru._3dbooruPopularExtractor, + "#pattern" : r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", + "#count" : 20, +}, + +) diff --git a/test/results/4chan.py b/test/results/4chan.py new file mode 100644 index 00000000..c90e4107 --- /dev/null +++ b/test/results/4chan.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.4chan") +_4chan = getattr(gallery_dl.extractor, "4chan") + + +__tests__ = ( +{ + "#url" : "https://boards.4chan.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class" : _4chan._4chanThreadExtractor, + "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + "#sha1_content" : "20b7b51afa51c9c31a0020a0737b889532c8d7ec", +}, + +{ + "#url" : "https://boards.4channel.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class" : _4chan._4chanThreadExtractor, + "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", +}, + +{ + "#url" : "https://boards.4channel.org/po/", + "#category": ("", "4chan", "board"), + "#class" : _4chan._4chanBoardExtractor, + "#pattern" : _4chan._4chanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +) diff --git a/test/results/4chanarchives.py b/test/results/4chanarchives.py new file mode 100644 index 00000000..8aa8befd --- /dev/null +++ b/test/results/4chanarchives.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.4chanarchives") +_4chanarchives = getattr(gallery_dl.extractor, "4chanarchives") + + +__tests__ = ( +{ + "#url" : "https://4chanarchives.com/board/c/thread/2707110", + "#category": ("", "4chanarchives", "thread"), + "#class" : _4chanarchives._4chanarchivesThreadExtractor, + "#pattern" : r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", + "#count" : 2, + + "board" : "c", + "com" : str, + "name" : "Anonymous", + "no" : int, + "thread": "2707110", + "time" : r"re:2016-07-1\d \d\d:\d\d:\d\d", + "title" : "Ren Kagami from 'Oyako Neburi'", +}, + +{ + "#url" : "https://4chanarchives.com/board/c/", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, + "#pattern" : _4chanarchives._4chanarchivesThreadExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "https://4chanarchives.com/board/c", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, +}, + +{ + "#url" : "https://4chanarchives.com/board/c/10", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, +}, + +) diff --git a/test/results/4plebs.py b/test/results/4plebs.py new file mode 100644 index 00000000..4f00027d --- /dev/null +++ b/test/results/4plebs.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archive.4plebs.org/tg/thread/54059290", + "#category": ("foolfuuka", "4plebs", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fd823f17b5001442b941fddcd9ec91bafedfbc79", +}, + +{ + "#url" : "https://archive.4plebs.org/tg/", + "#category": ("foolfuuka", "4plebs", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archive.4plebs.org/_/search/text/test/", + "#category": ("foolfuuka", "4plebs", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archive.4plebs.org/tg/gallery/1", + "#category": ("foolfuuka", "4plebs", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/500px.py b/test/results/500px.py new file mode 100644 index 00000000..39b618bc --- /dev/null +++ b/test/results/500px.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.500px") +_500px = getattr(gallery_dl.extractor, "500px") + + +__tests__ = ( +{ + "#url" : "https://500px.com/p/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, + "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", + "#range" : "1-99", + "#count" : 99, +}, + +{ + "#url" : "https://500px.com/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, +}, + +{ + "#url" : "https://web.500px.com/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, +}, + +{ + "#url" : "https://500px.com/p/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class" : _500px._500pxGalleryExtractor, + "#count" : 3, + "#sha1_url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", + + "gallery": dict, + "user" : dict, +}, + +{ + "#url" : "https://500px.com/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class" : _500px._500pxGalleryExtractor, +}, + +{ + "#url" : "https://500px.com/liked", + "#category": ("", "500px", "favorite"), + "#class" : _500px._500pxFavoriteExtractor, +}, + +{ + "#url" : "https://500px.com/photo/222049255/queen-of-coasts", + "#category": ("", "500px", "image"), + "#class" : _500px._500pxImageExtractor, + "#count" : 1, + "#sha1_url": "fbdf7df39325cae02f5688e9f92935b0e7113315", + + "camera" : "Canon EOS 600D", + "camera_info" : dict, + "comments" : list, + "comments_count" : int, + "created_at" : "2017-08-01T08:40:05+00:00", + "description" : str, + "editored_by" : None, + "editors_choice" : False, + "extension" : "jpg", + "feature" : "popular", + "feature_date" : "2017-08-01T09:58:28+00:00", + "focal_length" : "208", + "height" : 3111, + "id" : 222049255, + "image_format" : "jpg", + "image_url" : list, + "images" : list, + "iso" : "100", + "lens" : "EF-S55-250mm f/4-5.6 IS II", + "lens_info" : dict, + "liked" : None, + "location" : None, + "location_details": dict, + "name" : "Queen Of Coasts", + "nsfw" : False, + "privacy" : False, + "profile" : True, + "rating" : float, + "status" : 1, + "tags" : list, + "taken_at" : "2017-05-04T17:36:51+00:00", + "times_viewed" : int, + "url" : "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", + "user" : dict, + "user_id" : 12847235, + "votes_count" : int, + "watermark" : True, + "width" : 4637, +}, + +) diff --git a/test/results/8chan.py b/test/results/8chan.py new file mode 100644 index 00000000..43c08349 --- /dev/null +++ b/test/results/8chan.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.8chan") +_8chan = getattr(gallery_dl.extractor, "8chan") + + +__tests__ = ( +{ + "#url" : "https://8chan.moe/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, + "#pattern" : r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", + "#count" : 14, + + "archived" : False, + "autoSage" : False, + "boardDescription": "Film and Cinema", + "boardMarkdown" : None, + "boardName" : "Movies", + "boardUri" : "vhs", + "creation" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", + "cyclic" : False, + "email" : None, + "id" : r"re:^[0-9a-f]{6}$", + "locked" : False, + "markdown" : str, + "maxFileCount" : 5, + "maxFileSize" : "32.00 MB", + "maxMessageLength": 8001, + "message" : str, + "mime" : str, + "name" : "Anonymous", + "num" : int, + "originalName" : str, + "path" : r"re:/.media/[0-9a-f]{64}\.\w+$", + "pinned" : False, + "postId" : int, + "signedRole" : None, + "size" : int, + "threadId" : 4, + "thumb" : r"re:/.media/t_[0-9a-f]{64}$", + "uniquePosters" : 9, + "usesCustomCss" : True, + "usesCustomJs" : False, + "?wsPort" : 8880, + "?wssPort" : 2087, +}, + +{ + "#url" : "https://8chan.se/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, +}, + +{ + "#url" : "https://8chan.cc/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, +}, + +{ + "#url" : "https://8chan.moe/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +{ + "#url" : "https://8chan.moe/vhs/2.html", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, + "#pattern" : _8chan._8chanThreadExtractor.pattern, + "#count" : 23, +}, + +{ + "#url" : "https://8chan.se/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +{ + "#url" : "https://8chan.cc/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +) diff --git a/test/results/8kun.py b/test/results/8kun.py new file mode 100644 index 00000000..53d16e13 --- /dev/null +++ b/test/results/8kun.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://8kun.top/test/res/65248.html", + "#category": ("vichan", "8kun", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", + "#count" : ">= 8", +}, + +{ + "#url" : "https://8kun.top/v/index.html", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://8kun.top/v/2.html", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://8kun.top/v/index.html?PageSpeed=noscript", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/8muses.py b/test/results/8muses.py new file mode 100644 index 00000000..da19e4ef --- /dev/null +++ b/test/results/8muses.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.8muses") +_8muses = getattr(gallery_dl.extractor, "8muses") + + +__tests__ = ( +{ + "#url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#pattern" : r"https://comics.8muses.com/image/fl/[\w-]+", + "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", + + "url" : str, + "hash" : str, + "page" : int, + "count": 6, + "album": { + "id" : 10467, + "title" : "Liar", + "path" : "Fakku Comics/mogg/Liar", + "parts" : [ + "Fakku Comics", + "mogg", + "Liar", + ], + "private": False, + "url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "parent" : 10464, + "views" : int, + "likes" : int, + "date" : "dt:2018-07-10 00:00:00", + }, +}, + +{ + "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/santa", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#pattern" : _8muses._8musesAlbumExtractor.pattern, + "#count" : ">= 3", + + "url" : str, + "name" : str, + "private": False, +}, + +{ + "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", + "#comment" : "custom sorting", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#count" : ">= 70", + + "name": r"re:^[R-Zr-z]", +}, + +{ + "#url" : "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", + "#comment" : "non-ASCII characters", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#count" : 2, + + "name": r"re:From Trainers to Pokémons", +}, + +) diff --git a/test/results/94chan.py b/test/results/94chan.py new file mode 100644 index 00000000..6e792e30 --- /dev/null +++ b/test/results/94chan.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import jschan + + +__tests__ = ( +{ + "#url" : "https://94chan.org/art/thread/25.html", + "#category": ("jschan", "94chan", "thread"), + "#class" : jschan.JschanThreadExtractor, + "#pattern" : r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", + "#count" : ">= 15", +}, + +{ + "#url" : "https://94chan.org/art/", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, + "#pattern" : jschan.JschanThreadExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://94chan.org/art/2.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +{ + "#url" : "https://94chan.org/art/catalog.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +{ + "#url" : "https://94chan.org/art/index.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +) diff --git a/test/results/__init__.py b/test/results/__init__.py new file mode 100644 index 00000000..0fe87462 --- /dev/null +++ b/test/results/__init__.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import functools + +__directory__ = os.path.dirname(__file__) + + +@functools.lru_cache(maxsize=None) +def tests(name): + try: + module = __import__(name, globals(), None, (), 1) + return module.__tests__ + except Exception as exc: + print(exc) + return () + + +def all(): + ignore = ("__init__.py", "__pycache__") + for filename in os.listdir(__directory__): + if filename not in ignore: + yield from tests(filename[:-3]) + + +def category(category): + return tests(category.replace(".", "")) diff --git a/test/results/acidimg.py b/test/results/acidimg.py new file mode 100644 index 00000000..d61a7736 --- /dev/null +++ b/test/results/acidimg.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://acidimg.cc/img-5acb6b9de4640.html", + "#category": ("imagehost", "acidimg", "image"), + "#class" : imagehosts.AcidimgImageExtractor, + "#sha1_url" : "f132a630006e8d84f52d59555191ed82b3b64c04", + "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/adultempire.py b/test/results/adultempire.py new file mode 100644 index 00000000..1a2953ef --- /dev/null +++ b/test/results/adultempire.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import adultempire + + +__tests__ = ( +{ + "#url" : "https://www.adultempire.com/5998/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class" : adultempire.AdultempireGalleryExtractor, + "#range" : "1", + "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", + "#sha1_content" : "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", +}, + +{ + "#url" : "https://www.adultdvdempire.com/5683/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class" : adultempire.AdultempireGalleryExtractor, + "#sha1_url" : "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", + "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", +}, + +) diff --git a/test/results/aibooru.py b/test/results/aibooru.py new file mode 100644 index 00000000..78bd7273 --- /dev/null +++ b/test/results/aibooru.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://aibooru.online/posts?tags=center_frills&z=1", + "#category": ("Danbooru", "aibooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#pattern" : r"https://cdn\.aibooru\.online/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://safe.aibooru.online/posts?tags=center_frills", + "#category": ("Danbooru", "aibooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://aibooru.online/pools/1", + "#category": ("Danbooru", "aibooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, +}, + +{ + "#url" : "https://aibooru.online/posts/1", + "#category": ("Danbooru", "aibooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", +}, + +{ + "#url" : "https://aibooru.online/explore/posts/popular", + "#category": ("Danbooru", "aibooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/allgirlbooru.py b/test/results/allgirlbooru.py new file mode 100644 index 00000000..e1bfc11f --- /dev/null +++ b/test/results/allgirlbooru.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", + "#category": ("gelbooru_v01", "allgirlbooru", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", + "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", + "#category": ("gelbooru_v01", "allgirlbooru", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "b416800d2d2b072f80d3b37cfca9cb806fb25d51", + "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", + + "created_at": "2021-02-13 16:27:39", + "date" : "dt:2021-02-13 16:27:39", + "file_url" : "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", + "height" : "1200", + "id" : "107213", + "md5" : "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", + "rating" : "s", + "score" : str, + "source" : "", + "tags" : "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", + "uploader" : "Honochi31", + "width" : "1600", +}, + +) diff --git a/test/results/animereactor.py b/test/results/animereactor.py new file mode 100644 index 00000000..a83ac6ff --- /dev/null +++ b/test/results/animereactor.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://anime.reactor.cc/tag/Anime+Art", + "#category": ("reactor", "anime.reactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://anime.reactor.cc/user/Shuster", + "#category": ("reactor", "anime.reactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://anime.reactor.cc/post/3576250", + "#category": ("reactor", "anime.reactor", "post"), + "#class" : reactor.ReactorPostExtractor, +}, + +) diff --git a/test/results/architizer.py b/test/results/architizer.py new file mode 100644 index 00000000..3a751556 --- /dev/null +++ b/test/results/architizer.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import architizer + + +__tests__ = ( +{ + "#url" : "https://architizer.com/projects/house-lo/", + "#category": ("", "architizer", "project"), + "#class" : architizer.ArchitizerProjectExtractor, + "#pattern" : r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", + + "count" : 27, + "description": str, + "firm" : "Atelier Lina Bellovicova", + "gid" : "225496", + "location" : "Czechia", + "num" : int, + "size" : "1000 sqft - 3000 sqft", + "slug" : "house-lo", + "status" : "Built", + "subcategory": "project", + "title" : "House LO", + "type" : "Residential › Private House", + "year" : "2020", +}, + +{ + "#url" : "https://architizer.com/firms/olson-kundig/", + "#category": ("", "architizer", "firm"), + "#class" : architizer.ArchitizerFirmExtractor, + "#pattern" : architizer.ArchitizerProjectExtractor.pattern, + "#count" : ">= 90", +}, + +) diff --git a/test/results/archivedmoe.py b/test/results/archivedmoe.py new file mode 100644 index 00000000..90b48877 --- /dev/null +++ b/test/results/archivedmoe.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archived.moe/gd/thread/309639/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url" : "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", + "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", +}, + +{ + "#url" : "https://archived.moe/a/thread/159767162/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", +}, + +{ + "#url" : "https://archived.moe/gd/", + "#category": ("foolfuuka", "archivedmoe", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archived.moe/_/search/text/test/", + "#category": ("foolfuuka", "archivedmoe", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archived.moe/gd/gallery/2", + "#category": ("foolfuuka", "archivedmoe", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/archiveofsins.py b/test/results/archiveofsins.py new file mode 100644 index 00000000..3f3fcb93 --- /dev/null +++ b/test/results/archiveofsins.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archiveofsins.com/h/thread/4668813/", + "#category": ("foolfuuka", "archiveofsins", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url" : "f612d287087e10a228ef69517cf811539db9a102", + "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", +}, + +{ + "#url" : "https://archiveofsins.com/h/", + "#category": ("foolfuuka", "archiveofsins", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/h/gallery/3", + "#category": ("foolfuuka", "archiveofsins", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/artstation.py b/test/results/artstation.py new file mode 100644 index 00000000..262bad88 --- /dev/null +++ b/test/results/artstation.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import artstation +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.artstation.com/sungchoi/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, + "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/albums/all/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://sungchoi.artstation.com/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://sungchoi.artstation.com/projects/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://www.artstation.com/huimeiye/albums/770899", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.artstation.com/huimeiye/albums/770898", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://huimeiye.artstation.com/albums/770899", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, +}, + +{ + "#url" : "https://www.artstation.com/mikf/likes", + "#category": ("", "artstation", "likes"), + "#class" : artstation.ArtstationLikesExtractor, + "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#count" : 6, +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/likes", + "#comment" : "no likes", + "#category": ("", "artstation", "likes"), + "#class" : artstation.ArtstationLikesExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.artstation.com/contests/thu-2017/challenges/20", + "#category": ("", "artstation", "challenge"), + "#class" : artstation.ArtstationChallengeExtractor, +}, + +{ + "#url" : "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", + "#category": ("", "artstation", "challenge"), + "#class" : artstation.ArtstationChallengeExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.artstation.com/search?query=ancient&sort_by=rank", + "#category": ("", "artstation", "search"), + "#class" : artstation.ArtstationSearchExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.artstation.com/artwork?sorting=latest", + "#category": ("", "artstation", "artwork"), + "#class" : artstation.ArtstationArtworkExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.artstation.com/artwork/LQVJr", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#pattern" : r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", + "#sha1_content": "7b113871465fdc09d127adfdc2767d51cf45a7e9", +}, + +{ + "#url" : "https://www.artstation.com/artwork/Db3dy", + "#comment" : "multiple images per project", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://www.artstation.com/artwork/g4WPK", + "#comment" : "embedded youtube video", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#options" : {"external": True}, + "#pattern" : "ytdl:https://www.youtube.com/embed/JNFfJtwwrU0", + "#range" : "2", +}, + +{ + "#url" : "https://www.artstation.com/artwork/3q3mXB", + "#comment" : "404 (#3016)", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://sungchoi.artstation.com/projects/LQVJr", + "#comment" : "alternate URL patterns", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, +}, + +{ + "#url" : "https://artstn.co/p/LQVJr", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/following", + "#category": ("", "artstation", "following"), + "#class" : artstation.ArtstationFollowingExtractor, + "#pattern" : artstation.ArtstationUserExtractor.pattern, + "#count" : ">= 50", +}, + +) diff --git a/test/results/aryion.py b/test/results/aryion.py new file mode 100644 index 00000000..8b91e342 --- /dev/null +++ b/test/results/aryion.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import aryion + + +__tests__ = ( +{ + "#url" : "https://aryion.com/g4/gallery/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, + "#options" : {"recursive": False}, + "#pattern" : r"https://aryion\.com/g4/data\.php\?id=\d+$", + "#range" : "48-52", + "#count" : 5, +}, + +{ + "#url" : "https://aryion.com/g4/user/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, +}, + +{ + "#url" : "https://aryion.com/g4/latest.php?name=jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, +}, + +{ + "#url" : "https://aryion.com/g4/tags.php?tag=star+wars&p=19", + "#category": ("", "aryion", "tag"), + "#class" : aryion.AryionTagExtractor, + "#count" : ">= 5", +}, + +{ + "#url" : "https://aryion.com/g4/view/510079", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", + + "artist" : "jameshoward", + "user" : "jameshoward", + "filename" : "jameshoward-510079-subscribestar_150", + "extension" : "jpg", + "id" : 510079, + "width" : 1665, + "height" : 1619, + "size" : 784239, + "title" : "I'm on subscribestar now too!", + "description": r"re:Doesn't hurt to have a backup, right\?", + "tags" : [ + "Non-Vore", + "subscribestar", + ], + "date" : "dt:2019-02-16 19:30:34", + "path" : [], + "views" : int, + "favorites" : int, + "comments" : int, + "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", +}, + +{ + "#url" : "https://aryion.com/g4/view/588928", + "#comment" : "x-folder (#694)", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#pattern" : aryion.AryionPostExtractor.pattern, + "#count" : ">= 8", +}, + +{ + "#url" : "https://aryion.com/g4/view/537379", + "#comment" : "x-comic-folder (#945)", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#pattern" : aryion.AryionPostExtractor.pattern, + "#count" : 2, +}, + +) diff --git a/test/results/atfbooru.py b/test/results/atfbooru.py new file mode 100644 index 00000000..4bdd1b6d --- /dev/null +++ b/test/results/atfbooru.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://booru.allthefallen.moe/posts?tags=yume_shokunin", + "#category": ("Danbooru", "atfbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://booru.allthefallen.moe/pools/9", + "#category": ("Danbooru", "atfbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#count" : 6, + "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", +}, + +{ + "#url" : "https://booru.allthefallen.moe/posts/22", + "#category": ("Danbooru", "atfbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", +}, + +{ + "#url" : "https://booru.allthefallen.moe/explore/posts/popular", + "#category": ("Danbooru", "atfbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/b4k.py b/test/results/b4k.py new file mode 100644 index 00000000..5a2c13c2 --- /dev/null +++ b/test/results/b4k.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://arch.b4k.co/meta/thread/196/", + "#category": ("foolfuuka", "b4k", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", +}, + +{ + "#url" : "https://arch.b4k.co/meta/", + "#category": ("foolfuuka", "b4k", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://arch.b4k.co/meta/gallery/", + "#category": ("foolfuuka", "b4k", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/baraag.py b/test/results/baraag.py new file mode 100644 index 00000000..567d12c5 --- /dev/null +++ b/test/results/baraag.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://baraag.net/@pumpkinnsfw", + "#category": ("mastodon", "baraag", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://baraag.net/bookmarks", + "#category": ("mastodon", "baraag", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://baraag.net/users/pumpkinnsfw/following", + "#category": ("mastodon", "baraag", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://baraag.net/@pumpkinnsfw/104364170556898443", + "#category": ("mastodon", "baraag", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", +}, + +) diff --git a/test/results/bbc.py b/test/results/bbc.py new file mode 100644 index 00000000..83b4923c --- /dev/null +++ b/test/results/bbc.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import bbc + + +__tests__ = ( +{ + "#url" : "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", + "#category": ("", "bbc", "gallery"), + "#class" : bbc.BbcGalleryExtractor, + "#pattern" : r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", + "#count" : 37, + + "programme": "p084qtzs", + "path" : [ + "BBC One", + "Doctor Who", + "The Timeless Children", + ], +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/p084qtzs", + "#category": ("", "bbc", "gallery"), + "#class" : bbc.BbcGalleryExtractor, +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries", + "#category": ("", "bbc", "programme"), + "#class" : bbc.BbcProgrammeExtractor, + "#pattern" : bbc.BbcGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", + "#category": ("", "bbc", "programme"), + "#class" : bbc.BbcProgrammeExtractor, + "#pattern" : bbc.BbcGalleryExtractor.pattern, + "#count" : ">= 100", +}, + +) diff --git a/test/results/bbw-chan.py b/test/results/bbw-chan.py new file mode 100644 index 00000000..74913ff9 --- /dev/null +++ b/test/results/bbw-chan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://bbw-chan.nl/bbwdraw/res/499.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://bbw-chan\.nl/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count" : ">= 352", +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/res/489.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 148", +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/2.html", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/bcbnsfw.py b/test/results/bcbnsfw.py new file mode 100644 index 00000000..8d35e36e --- /dev/null +++ b/test/results/bcbnsfw.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import szurubooru + + +__tests__ = ( +{ + "#url" : "https://booru.bcbnsfw.space/posts/query=simple_background", + "#category": ("szurubooru", "bcbnsfw", "tag"), + "#class" : szurubooru.SzurubooruTagExtractor, +}, + +{ + "#url" : "https://booru.bcbnsfw.space/post/1599", + "#category": ("szurubooru", "bcbnsfw", "post"), + "#class" : szurubooru.SzurubooruPostExtractor, + "#pattern" : r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", + "#sha1_content": "0c38fc612ba1f03950fad31c4f80a1fccdab1096", +}, + +) diff --git a/test/results/behance.py b/test/results/behance.py new file mode 100644 index 00000000..7b39b5b4 --- /dev/null +++ b/test/results/behance.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import behance +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.behance.net/gallery/17386197/A-Short-Story", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#count" : 2, + "#sha1_url": "ab79bd3bef8d3ae48e6ac74fd995c1dfaec1b7d2", + + "id" : 17386197, + "name" : r"re:\"Hi\". A short story about the important things ", + "owners": [ + "Place Studio", + "Julio César Velazquez", + ], + "fields": [ + "Animation", + "Character Design", + "Directing", + ], + "tags" : list, + "module": dict, + "date" : "dt:2014-06-03 15:41:51", +}, + +{ + "#url" : "https://www.behance.net/gallery/21324767/Nevada-City", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#count" : 6, + "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", + + "owners": ["Alex Strohl"], +}, + +{ + "#url" : "https://www.behance.net/gallery/88276087/Audi-R8-RWD", + "#comment" : "'media_collection' modules", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#pattern" : r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", + "#count" : 20, + "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", +}, + +{ + "#url" : "https://www.behance.net/gallery/101185577/COLCCI", + "#comment" : "'video' modules (#1282)", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#pattern" : r"https://cdn-prod-ccv\.adobe\.com/\w+/rend/\w+_720\.mp4\?", + "#count" : 3, +}, + +{ + "#url" : "https://www.behance.net/gallery/177464639/Kimori", + "#comment" : "mature content (#4417)", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://www.behance.net/alexstrohl", + "#category": ("", "behance", "user"), + "#class" : behance.BehanceUserExtractor, + "#pattern" : behance.BehanceGalleryExtractor.pattern, + "#count" : ">= 11", +}, + +{ + "#url" : "https://www.behance.net/collection/71340149/inspiration", + "#category": ("", "behance", "collection"), + "#class" : behance.BehanceCollectionExtractor, + "#pattern" : behance.BehanceGalleryExtractor.pattern, + "#count" : ">= 150", +}, + +) diff --git a/test/results/bitly.py b/test/results/bitly.py new file mode 100644 index 00000000..cbfcf0eb --- /dev/null +++ b/test/results/bitly.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import urlshortener + + +__tests__ = ( +{ + "#url" : "https://bit.ly/3cWIUgq", + "#category": ("urlshortener", "bitly", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#pattern" : "^https://gumroad.com/l/storm_b1", + "#count" : 1, +}, + +) diff --git a/test/results/blogger.py b/test/results/blogger.py new file mode 100644 index 00000000..214d450d --- /dev/null +++ b/test/results/blogger.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import blogger + + +__tests__ = ( +{ + "#url" : "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : "https://3.bp.blogspot.com/.*/s0/Icy-Moonrise-.*.jpg", + "#sha1_url": "9928429fb62f712eb4de80f53625eccecc614aae", + + "blog": { + "date" : "dt:2010-11-21 18:19:42", + "description": "", + "id" : "5623928067739466034", + "kind" : "blogger#blog", + "locale" : dict, + "name" : "Julian Bunker Photography", + "pages" : int, + "posts" : int, + "published" : "2010-11-21T10:19:42-08:00", + "updated" : str, + "url" : "http://julianbphotography.blogspot.com/", + }, + "post": { + "author" : "Julian Bunker", + "content" : str, + "date" : "dt:2010-12-26 01:08:00", + "etag" : str, + "id" : "6955139236418998998", + "kind" : "blogger#post", + "published": "2010-12-25T17:08:00-08:00", + "replies" : "0", + "title" : "Moon Rise", + "updated" : "2011-12-06T05:21:24-08:00", + "url" : r"re:.+/2010/12/moon-rise.html$", + }, + "num" : int, + "url" : str, +}, + +{ + "#url" : "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, +}, + +{ + "#url" : "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", + "#comment" : "video (#587)", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : r"https://.+\.googlevideo\.com/videoplayback", +}, + +{ + "#url" : "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", + "#comment" : "new image domain (#2204)", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : "https://blogger.googleusercontent.com/img/a/.+=s0$", + "#count" : 8, +}, + +{ + "#url" : "https://julianbphotography.blogspot.com/", + "#category": ("", "blogger", "blog"), + "#class" : blogger.BloggerBlogExtractor, + "#pattern" : r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg", + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "blogger:https://www.kefblog.com.ng/", + "#category": ("", "blogger", "blog"), + "#class" : blogger.BloggerBlogExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://julianbphotography.blogspot.com/search?q=400mm", + "#category": ("", "blogger", "search"), + "#class" : blogger.BloggerSearchExtractor, + "#count" : "< 10", + + "query": "400mm", +}, + +{ + "#url" : "https://dmmagazine.blogspot.com/search/label/D%26D", + "#category": ("", "blogger", "label"), + "#class" : blogger.BloggerLabelExtractor, + "#range" : "1-25", + "#count" : 25, + + "label": "D&D", +}, + +) diff --git a/test/results/booruvar.py b/test/results/booruvar.py new file mode 100644 index 00000000..8beb45e1 --- /dev/null +++ b/test/results/booruvar.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://booru.borvar.art/posts?tags=chibi&z=1", + "#category": ("Danbooru", "booruvar", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#pattern" : r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://booru.borvar.art/pools/2", + "#category": ("Danbooru", "booruvar", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#count" : 4, + "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", +}, + +{ + "#url" : "https://booru.borvar.art/posts/1487", + "#category": ("Danbooru", "booruvar", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", +}, + +{ + "#url" : "https://booru.borvar.art/explore/posts/popular", + "#category": ("Danbooru", "booruvar", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/bunkr.py b/test/results/bunkr.py new file mode 100644 index 00000000..5a66d02c --- /dev/null +++ b/test/results/bunkr.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import bunkr + + +__tests__ = ( +{ + "#url" : "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://cdn\.bunkr\.ru/test-テスト-\"&>-QjgneIQv\.png", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "album_id" : "Lktg9Keq", + "album_name": "test テスト \"&>", + "count" : 1, + "filename" : "test-テスト-\"&>-QjgneIQv", + "id" : "QjgneIQv", + "name" : "test-テスト-\"&>", + "num" : int, +}, + +{ + "#url" : "https://app.bunkr.ru/a/ptRHaCn2", + "#comment" : "mp4 (#2239)", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://media-files\.bunkr\.ru/_-RnHoW69L\.mp4", + "#sha1_content": "80e61d1dbc5896ae7ef9a28734c747b28b320471", +}, + +{ + "#url" : "https://bunkr.is/a/iXTTc1o2", + "#comment" : "cdn4", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://(cdn|media-files)4\.bunkr\.ru/", + "#sha1_content": "da29aae371b7adc8c5ef8e6991b66b69823791e8", + + "album_id" : "iXTTc1o2", + "album_name" : "test2", + "album_size" : "691.1 KB", + "count" : 2, + "description": "072022", + "filename" : r"re:video-wFO9FtxG|image-sZrQUeOx", + "id" : r"re:wFO9FtxG|sZrQUeOx", + "name" : r"re:video|image", + "num" : int, +}, + +{ + "#url" : "https://bunkrr.su/a/j1G29CnD", + "#comment" : "cdn12 .ru TLD (#4147)", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://(cdn12.bunkr.ru|media-files12.bunkr.la)/\w+", + "#count" : 8, +}, + +{ + "#url" : "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.la/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.is/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.to/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +) diff --git a/test/results/catbox.py b/test/results/catbox.py new file mode 100644 index 00000000..b977a52e --- /dev/null +++ b/test/results/catbox.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import catbox + + +__tests__ = ( +{ + "#url" : "https://catbox.moe/c/1igcbe", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, + "#pattern" : r"https://files\.catbox\.moe/\w+\.\w{3}$", + "#count" : 3, + "#sha1_url" : "35866a88c29462814f103bc22ec031eaeb380f8a", + "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", + + "album_id" : "1igcbe", + "album_name" : "test", + "date" : "dt:2022-08-18 00:00:00", + "description": "album test &>", +}, + +{ + "#url" : "https://www.catbox.moe/c/cd90s1", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, +}, + +{ + "#url" : "https://catbox.moe/c/w7tm47#", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, +}, + +{ + "#url" : "https://files.catbox.moe/8ih3y7.png", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, + "#pattern" : r"^https://files\.catbox\.moe/8ih3y7\.png$", + "#count" : 1, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://litter.catbox.moe/t8v3n9.png", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, +}, + +{ + "#url" : "https://de.catbox.moe/bjdmz1.jpg", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, +}, + +) diff --git a/test/results/cavemanon.py b/test/results/cavemanon.py new file mode 100644 index 00000000..4fc0af3c --- /dev/null +++ b/test/results/cavemanon.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://booru.cavemanon.xyz/post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, +}, + +{ + "#url" : "https://booru.cavemanon.xyz/index.php?q=post/view/8335", + "#category": ("shimmie2", "cavemanon", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", + "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", + + "extension": "png", + "file_url" : "https://booru.cavemanon.xyz/index.php?q=image/8335.png", + "filename" : "8335", + "height" : 460, + "id" : 8335, + "md5" : "", + "size" : 0, + "tags" : "Color Fang Food Pterodactyl discord_emote transparent", + "width" : 459, +}, + +) diff --git a/test/results/chelseacrew.py b/test/results/chelseacrew.py new file mode 100644 index 00000000..7c12613f --- /dev/null +++ b/test/results/chelseacrew.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://chelseacrew.com/collections/flats", + "#category": ("shopify", "chelseacrew", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://chelseacrew.com/collections/flats/products/dora", + "#category": ("shopify", "chelseacrew", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/comicvine.py b/test/results/comicvine.py new file mode 100644 index 00000000..fa77c491 --- /dev/null +++ b/test/results/comicvine.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import comicvine + + +__tests__ = ( +{ + "#url" : "https://comicvine.gamespot.com/jock/4040-5653/images/", + "#category": ("", "comicvine", "tag"), + "#class" : comicvine.ComicvineTagExtractor, + "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", + "#count" : ">= 140", +}, + +{ + "#url" : "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", + "#category": ("", "comicvine", "tag"), + "#class" : comicvine.ComicvineTagExtractor, + "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", + "#count" : ">= 450", +}, + +) diff --git a/test/results/coomerparty.py b/test/results/coomerparty.py new file mode 100644 index 00000000..dfc4a188 --- /dev/null +++ b/test/results/coomerparty.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kemonoparty + + +__tests__ = ( +{ + "#url" : "https://coomer.party/onlyfans/user/alinity/post/125962203", + "#comment" : "coomer.party (#2100)", + "#category": ("", "coomerparty", "onlyfans"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg", +}, + +) diff --git a/test/results/cyberdrop.py b/test/results/cyberdrop.py new file mode 100644 index 00000000..b163ca47 --- /dev/null +++ b/test/results/cyberdrop.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import cyberdrop + + +__tests__ = ( +{ + "#url" : "https://cyberdrop.me/a/keKRjm4t", + "#comment" : "images", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class" : cyberdrop.CyberdropAlbumExtractor, + "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$", + + "album_id" : "keKRjm4t", + "album_name" : "Fate (SFW)", + "album_size" : 150069254, + "count" : 62, + "date" : "dt:2020-06-18 13:14:20", + "description": "", + "id" : r"re:\w{8}", +}, + +{ + "#url" : "https://cyberdrop.to/a/l8gIAXVD", + "#comment" : "videos", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class" : cyberdrop.CyberdropAlbumExtractor, + "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$", + "#count" : 31, + + "album_id" : "l8gIAXVD", + "album_name": "Achelois17 videos", + "album_size": 652037121, + "date" : "dt:2020-06-16 15:40:44", +}, + +) diff --git a/test/results/danbooru.py b/test/results/danbooru.py new file mode 100644 index 00000000..c64c693e --- /dev/null +++ b/test/results/danbooru.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://danbooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts?tags=mushishi", + "#comment" : "test page transitions", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#count" : ">= 300", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", + "#comment" : "'external' option (#1747)", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#options" : {"external": True}, + "#pattern" : r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", +}, + +{ + "#url" : "https://hijiribe.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://sonohara.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://safebooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/pools/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", +}, + +{ + "#url" : "https://danbooru.donmai.us/pool/show/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/posts/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + + "date": "dt:2008-08-12 04:46:05", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts/3613024", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#options" : {"ugoira": True}, + "#pattern" : r"https?://.+\.zip$", +}, + +{ + "#url" : "https://danbooru.donmai.us/post/show/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/explore/posts/popular", + "#category": ("Danbooru", "danbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", + "#category": ("Danbooru", "danbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, + "#range" : "1-120", + "#count" : 120, +}, + +) diff --git a/test/results/derpibooru.py b/test/results/derpibooru.py new file mode 100644 index 00000000..7ca6a0de --- /dev/null +++ b/test/results/derpibooru.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#count" : 1, + "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", + + "animated" : False, + "aspect_ratio" : 1.0, + "comment_count" : int, + "created_at" : "2012-01-02T03:12:33Z", + "date" : "dt:2012-01-02 03:12:33", + "deletion_reason" : None, + "description" : "", + "downvotes" : int, + "duplicate_of" : None, + "duration" : 0.04, + "extension" : "png", + "faves" : int, + "first_seen_at" : "2012-01-02T03:12:33Z", + "format" : "png", + "height" : 900, + "hidden_from_users": False, + "id" : 1, + "mime_type" : "image/png", + "name" : "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", + "orig_sha512_hash": None, + "processed" : True, + "representations" : dict, + "score" : int, + "sha512_hash" : "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", + "size" : 860914, + "source_url" : "https://www.deviantart.com/speccysy/art/Afternoon-Flight-215193985", + "spoilered" : False, + "tag_count" : int, + "tag_ids" : list, + "tags" : list, + "thumbnails_generated": True, + "updated_at" : r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", + "uploader" : "Clover the Clever", + "uploader_id" : 211188, + "upvotes" : int, + "view_url" : str, + "width" : 900, + "wilson_score" : float, +}, + +{ + "#url" : "https://derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://www.derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://www.derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://derpibooru.org/search?q=cute", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://derpibooru.org/tags/cute", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://derpibooru.org/galleries/1", + "#category": ("philomena", "derpibooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#pattern" : r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", + + "gallery": { + "description" : "Indexes start at 1 :P", + "id" : 1, + "spoiler_warning": "", + "thumbnail_id" : 1, + "title" : "The Very First Gallery", + "user" : "DeliciousBlackInk", + "user_id" : 365446, + }, +}, + +) diff --git a/test/results/desktopography.py b/test/results/desktopography.py new file mode 100644 index 00000000..c6066692 --- /dev/null +++ b/test/results/desktopography.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import desktopography + + +__tests__ = ( +{ + "#url" : "https://desktopography.net/", + "#category": ("", "desktopography", "site"), + "#class" : desktopography.DesktopographySiteExtractor, +}, + +{ + "#url" : "https://desktopography.net/exhibition-2020/", + "#category": ("", "desktopography", "exhibition"), + "#class" : desktopography.DesktopographyExhibitionExtractor, +}, + +{ + "#url" : "https://desktopography.net/portfolios/new-era/", + "#category": ("", "desktopography", "entry"), + "#class" : desktopography.DesktopographyEntryExtractor, +}, + +) diff --git a/test/results/desuarchive.py b/test/results/desuarchive.py new file mode 100644 index 00000000..8e180652 --- /dev/null +++ b/test/results/desuarchive.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://desuarchive.org/a/thread/159542679/", + "#category": ("foolfuuka", "desuarchive", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", +}, + +{ + "#url" : "https://desuarchive.org/a/", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://desuarchive.org/_/search/text/test/", + "#category": ("foolfuuka", "desuarchive", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://desuarchive.org/a/gallery/5", + "#category": ("foolfuuka", "desuarchive", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/deviantart.py b/test/results/deviantart.py new file mode 100644 index 00000000..87473bb8 --- /dev/null +++ b/test/results/deviantart.py @@ -0,0 +1,732 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import deviantart +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, + "#pattern" : "/shimoda7/gallery$", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$", + "#count" : 5, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#pattern" : r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", + "#count" : ">= 30", + + "allows_comments" : bool, + "author" : { + "type" : "regular", + "usericon": str, + "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "category_path" : str, + "content" : { + "filesize" : int, + "height" : int, + "src" : str, + "transparency": bool, + "width" : int, + }, + "da_category" : str, + "date" : datetime.datetime, + "deviationid" : str, + "?download_filesize": int, + "extension" : str, + "index" : int, + "is_deleted" : bool, + "is_downloadable" : bool, + "is_favourited" : bool, + "is_mature" : bool, + "preview" : { + "height" : int, + "src" : str, + "transparency": bool, + "width" : int, + }, + "published_time" : int, + "stats" : { + "comments" : int, + "favourites": int, + }, + "target" : dict, + "thumbs" : list, + "title" : str, + "url" : r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", + "username" : "shimoda7", +}, + +{ + "#url" : "https://www.deviantart.com/yakuzafc/gallery", + "#comment" : "group", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#pattern" : r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.deviantart.com/justatest235723/gallery", + "#comment" : "'folders' option (#276)", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#options" : { + "metadata": 1, + "folders" : 1, + "original": 0, + }, + "#count" : 3, + + "description": str, + "folders" : list, + "is_watching": bool, + "license" : str, + "tags" : list, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda8/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/all", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/all/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", + "#comment" : "user", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 5, +}, + +{ + "#url" : "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", + "#comment" : "group", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : ">= 4", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", + "#comment" : "uuid", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 5, +}, + +{ + "#url" : "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", + "#comment" : "name starts with '_', special characters (#1451)", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 1, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, +}, + +{ + "#url" : "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, +}, + +{ + "#url" : "https://sta.sh/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.png\?token=.+", + "#count" : 1, + "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", +}, + +{ + "#url" : "https://sta.sh/21jf51j7pzl2", + "#comment" : "multiple stash items", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#options" : {"original": False}, + "#count" : 4, +}, + +{ + "#url" : "https://sta.sh/024t4coz16mi", + "#comment" : "downloadable, but no 'content' field (#307)", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", + "#count" : 1, +}, + +{ + "#url" : "https://sta.sh/215twi387vfj", + "#comment" : "mixed folders and images (#659)", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#options" : {"original": False}, + "#count" : 4, +}, + +{ + "#url" : "https://sta.sh/abcdefghijkl", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, + "#options" : { + "metadata": True, + "flat" : False, + }, + "#count" : 1, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, + "#options" : {"original": False}, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, + "#options" : {"original": False}, + "#count" : ">= 15", +}, + +{ + "#url" : "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#sha1_url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#options" : {"journals": "text"}, + "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#options" : {"journals": "none"}, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/posts/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/journal/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/t1na/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#count" : 4, + "#sha1_url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074", +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : {"journals": "none"}, + "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/f/[^/]+/[^.]+\.jpg\?token=", + "#count" : 1, +}, + +{ + "#url" : "https://www.deviantart.com/vanillaghosties/posts/statuses", + "#comment" : "shared sta.sh item", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : { + "journals": "none", + "original": False, + }, + "#range" : "5-", + "#count" : 1, + + "index" : int, + "index_base36": r"re:^[0-9a-z]+$", + "url" : r"re:^https://sta.sh", +}, + +{ + "#url" : "https://www.deviantart.com/AndrejSKalin/posts/statuses", + "#comment" : "'deleted' deviations in 'items'", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : { + "journals" : "none", + "original" : 0, + "image-filter": "deviationid[:8] == '147C8B03'", + }, + "#count" : 2, + "#archive" : False, + + "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : {"journals": "text"}, + "#sha1_url": "c8744f7f733a3029116607b826321233c5ca452d", +}, + +{ + "#url" : "https://www.deviantart.com/?order=popular-all-time", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/popular-24-hours/?q=tree+house", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/artisan/popular-all-time/?q=tree", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/tag/nature", + "#category": ("", "deviantart", "tag"), + "#class" : deviantart.DeviantartTagExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/watch/deviations", + "#category": ("", "deviantart", "watch"), + "#class" : deviantart.DeviantartWatchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/notifications/watch", + "#category": ("", "deviantart", "watch"), + "#class" : deviantart.DeviantartWatchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/watch/posts", + "#category": ("", "deviantart", "watch-posts"), + "#class" : deviantart.DeviantartWatchPostsExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"comments": True}, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", + + "comments": list, +}, + +{ + "#url" : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", + "#comment" : "wixmp URL rewrite", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/f/[^/]+/[^.]+\.jpg\?token=", +}, + +{ + "#url" : "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", + "#comment" : "GIF (#242)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", +}, + +{ + "#url" : "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", + "#comment" : "Flash animation with GIF preview (#1731)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", + + "filename" : "flash_comic_tutorial_by_yuumei-d3juatd", + "extension": "swf", +}, + +{ + "#url" : "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", + "#comment" : "sta.sh URLs from description (#302)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : { + "extra" : 1, + "original": 0, + }, + "#pattern" : deviantart.DeviantartStashExtractor.pattern, + "#range" : "2-", + "#count" : 4, +}, + +{ + "#url" : "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", + "#comment" : "sta.sh URL from deviation['text_content']['body']['features']", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"extra": 1}, + "#pattern" : r"""text:<!DOCTYPE html> +|(?:https?://)?sta\.sh/([a-z0-9]+)""", + "#count" : 2, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", + "#comment" : "journal", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : """text:<!DOCTYPE html> +""", + "#sha1_url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", +}, + +{ + "#url" : "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", + "#comment" : "journal-like post with isJournal == False (#419)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : """text:<!DOCTYPE html> +""", + "#sha1_url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", +}, + +{ + "#url" : "https://deviantart.com/view/904858796/", + "#comment" : "/view/ URLs", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", +}, + +{ + "#url" : "http://www.deviantart.com/view/890672057", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", +}, + +{ + "#url" : "https://www.deviantart.com/view/706871727", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65", +}, + +{ + "#url" : "https://www.deviantart.com/view/1", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/deviation/817215762", + "#comment" : "/deviation/ (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://fav.me/ddijrpu", + "#comment" : "fav.me (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://fav.me/dddd", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", + "#comment" : "old-style URLs", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://zzz.deviantart.com/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/view.php?id=14864502", + "#comment" : "old /view/ URLs from the Wayback Machine", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "http://www.deviantart.com/view-full.php?id=100842", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.fxdeviantart.com/view/1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/search?q=tree", + "#category": ("", "deviantart", "search"), + "#class" : deviantart.DeviantartSearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/search/deviations?order=popular-1-week", + "#category": ("", "deviantart", "search"), + "#class" : deviantart.DeviantartSearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory", + "#category": ("", "deviantart", "gallery-search"), + "#class" : deviantart.DeviantartGallerySearchExtractor, + "#options" : {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", + "#category": ("", "deviantart", "gallery-search"), + "#class" : deviantart.DeviantartGallerySearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/about#watching", + "#category": ("", "deviantart", "following"), + "#class" : deviantart.DeviantartFollowingExtractor, + "#pattern" : deviantart.DeviantartUserExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +) diff --git a/test/results/directlink.py b/test/results/directlink.py new file mode 100644 index 00000000..afa43b34 --- /dev/null +++ b/test/results/directlink.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import directlink + + +__tests__ = ( +{ + "#url" : "https://en.wikipedia.org/static/images/project-logos/enwiki.png", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "18c5d00077332e98e53be9fed2ee4be66154b88d", + "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", +}, + +{ + "#url" : "https://example.org/file.webm", + "#comment" : "empty path", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "2d807ed7059d1b532f1bb71dc24b510b80ff943f", + "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", +}, + +{ + "#url" : "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", + "#comment" : "more complex example", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "6fb1061390f8aada3db01cb24b51797c7ee42b31", + "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", +}, + +{ + "#url" : "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", + "#comment" : "percent-encoded characters", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "2627e8140727fdf743f86fe18f69f99a052c9718", + "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", +}, + +{ + "#url" : "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", + "#comment" : "upper case file extension (#296)", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, +}, + +{ + "#url" : "https://räksmörgås.josefsson.org/raksmorgas.jpg", + "#comment" : "internationalized domain name", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "a65667f670b194afbd1e3ea5e7a78938d36747da", + "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", +}, + +) diff --git a/test/results/drawfriends.py b/test/results/drawfriends.py new file mode 100644 index 00000000..d481fe70 --- /dev/null +++ b/test/results/drawfriends.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "drawfriends", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, +}, + +{ + "#url" : "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "drawfriends", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, +}, + +{ + "#url" : "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", + "#category": ("gelbooru_v01", "drawfriends", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, +}, + +) diff --git a/test/results/dynastyscans.py b/test/results/dynastyscans.py new file mode 100644 index 00000000..bcd85917 --- /dev/null +++ b/test/results/dynastyscans.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import dynastyscans + + +__tests__ = ( +{ + "#url" : "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", + "#category": ("", "dynastyscans", "chapter"), + "#class" : dynastyscans.DynastyscansChapterExtractor, + "#sha1_url" : "dce64e8c504118f1ab4135c00245ea12413896cb", + "#sha1_metadata": "b67599703c27316a2fe4f11c3232130a1904e032", +}, + +{ + "#url" : "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", + "#category": ("", "dynastyscans", "chapter"), + "#class" : dynastyscans.DynastyscansChapterExtractor, + "#sha1_url" : "dbe5bbb74da2edcfb1832895a484e2a40bc8b538", + "#sha1_metadata": "6b674eb3a274999153f6be044973b195008ced2f", +}, + +{ + "#url" : "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", + "#category": ("", "dynastyscans", "manga"), + "#class" : dynastyscans.DynastyscansMangaExtractor, + "#pattern" : dynastyscans.DynastyscansChapterExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://dynasty-scans.com/images?with[]=4930&with[]=5211", + "#category": ("", "dynastyscans", "search"), + "#class" : dynastyscans.DynastyscansSearchExtractor, + "#sha1_url" : "22cf0fb64e12b29e79b0a3d26666086a48f9916a", + "#sha1_metadata": "11cbc555a15528d25567977b8808e10369c4c3ee", +}, + +{ + "#url" : "https://dynasty-scans.com/images", + "#category": ("", "dynastyscans", "search"), + "#class" : dynastyscans.DynastyscansSearchExtractor, + "#range" : "1", + "#count" : 1, +}, + +{ + "#url" : "https://dynasty-scans.com/images/1245", + "#category": ("", "dynastyscans", "image"), + "#class" : dynastyscans.DynastyscansImageExtractor, + "#sha1_url" : "15e54bd94148a07ed037f387d046c27befa043b2", + "#sha1_metadata": "0d8976c2d6fbc9ed6aa712642631b96e456dc37f", +}, + +) diff --git a/test/results/e621.py b/test/results/e621.py new file mode 100644 index 00000000..dd9787cd --- /dev/null +++ b/test/results/e621.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e621.net/posts?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, + "#sha1_url" : "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", +}, + +{ + "#url" : "https://e621.net/post/index/1/anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e621.net/post?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e621.net/pools/73", + "#category": ("E621", "e621", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url" : "1bd09a72715286a79eea3b7f09f51b3493eb579a", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", +}, + +{ + "#url" : "https://e621.net/pool/show/73", + "#category": ("E621", "e621", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e621.net/posts/535", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + + "date": "dt:2007-02-17 19:02:32", +}, + +{ + "#url" : "https://e621.net/posts/3181052", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, + "#options" : {"metadata": "notes,pools"}, + "#pattern" : r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", + + "notes": [{ + "body" : "Little Legends 2", + "created_at" : "2022-05-16T13:58:38.877-04:00", + "creator_id" : 517450, + "creator_name": "EeveeCuddler69", + "height" : 475, + "id" : 321296, + "is_active" : True, + "post_id" : 3181052, + "updated_at" : "2022-05-16T13:59:02.050-04:00", + "version" : 3, + "width" : 809, + "x" : 83, + "y" : 117, +}], + "pools": [{ + "category" : "series", + "created_at" : "2022-02-17T00:29:22.669-05:00", + "creator_id" : 1077440, + "creator_name": "Yeetus90", + "description" : """* "Little Legends":/pools/27971 +* Little Legends 2 +* "Little Legends 3":/pools/27481""", + "id" : 27492, + "is_active" : False, + "name" : "Little Legends 2", + "post_count" : 39, + "post_ids" : list, + "updated_at" : "2022-03-27T06:30:03.382-04:00", +}], +}, + +{ + "#url" : "https://e621.net/post/show/535", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e621.net/explore/posts/popular", + "#category": ("E621", "e621", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e621", "popular"), + "#class" : e621.E621PopularExtractor, + "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count" : ">= 70", +}, + +{ + "#url" : "https://e621.net/favorites", + "#category": ("E621", "e621", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +{ + "#url" : "https://e621.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e621", "favorite"), + "#class" : e621.E621FavoriteExtractor, + "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count" : "> 260", +}, + +) diff --git a/test/results/e6ai.py b/test/results/e6ai.py new file mode 100644 index 00000000..291f3b3a --- /dev/null +++ b/test/results/e6ai.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e6ai.net/posts?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/post/index/1/anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/post?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/pools/3", + "#category": ("E621", "e6ai", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", +}, + +{ + "#url" : "https://e6ai.net/pool/show/3", + "#category": ("E621", "e6ai", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e6ai.net/posts/23", + "#category": ("E621", "e6ai", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "3c85a806b3d9eec861948af421fe0e8ad6b8f881", + "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", +}, + +{ + "#url" : "https://e6ai.net/post/show/23", + "#category": ("E621", "e6ai", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e6ai.net/explore/posts/popular", + "#category": ("E621", "e6ai", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e6ai.net/favorites", + "#category": ("E621", "e6ai", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +) diff --git a/test/results/e926.py b/test/results/e926.py new file mode 100644 index 00000000..b046b459 --- /dev/null +++ b/test/results/e926.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e926.net/posts?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, + "#sha1_url" : "12198b275c62ffe2de67cca676c8e64de80c425d", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", +}, + +{ + "#url" : "https://e926.net/post/index/1/anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e926.net/post?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e926.net/pools/73", + "#category": ("E621", "e926", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url" : "6936f1b6a18c5c25bee7cad700088dbc2503481b", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", +}, + +{ + "#url" : "https://e926.net/pool/show/73", + "#category": ("E621", "e926", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e926.net/posts/535", + "#category": ("E621", "e926", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", +}, + +{ + "#url" : "https://e926.net/post/show/535", + "#category": ("E621", "e926", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e926.net/explore/posts/popular", + "#category": ("E621", "e926", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e926", "popular"), + "#class" : e621.E621PopularExtractor, + "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count" : ">= 70", +}, + +{ + "#url" : "https://e926.net/favorites", + "#category": ("E621", "e926", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +{ + "#url" : "https://e926.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e926", "favorite"), + "#class" : e621.E621FavoriteExtractor, + "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count" : "> 260", +}, + +) diff --git a/test/results/endchan.py b/test/results/endchan.py new file mode 100644 index 00000000..293a8569 --- /dev/null +++ b/test/results/endchan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://endchan.org/yuri/res/193483.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://endchan\.org/\.media/[^.]+(\.\w+)?$", + "#count" : ">= 19", +}, + +{ + "#url" : "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, +}, + +{ + "#url" : "https://endchan.org/yuri/", + "#category": ("lynxchan", "endchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 9", +}, + +{ + "#url" : "https://endchan.org/yuri/catalog.html", + "#category": ("lynxchan", "endchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/erome.py b/test/results/erome.py new file mode 100644 index 00000000..f3269c04 --- /dev/null +++ b/test/results/erome.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import erome + + +__tests__ = ( +{ + "#url" : "https://www.erome.com/a/NQgdlWvk", + "#category": ("", "erome", "album"), + "#class" : erome.EromeAlbumExtractor, + "#pattern" : r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", + "#count" : 1, + + "album_id": "NQgdlWvk", + "num" : 1, + "title" : "porn", + "user" : "yYgWBZw8o8qsMzM", +}, + +{ + "#url" : "https://www.erome.com/a/TdbZ4ogi", + "#category": ("", "erome", "album"), + "#class" : erome.EromeAlbumExtractor, + "#pattern" : r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", + "#count" : 6, + + "album_id": "TdbZ4ogi", + "num" : int, + "title" : "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", + "user" : "yYgWBZw8o8qsMzM", +}, + +{ + "#url" : "https://www.erome.com/yYgWBZw8o8qsMzM", + "#category": ("", "erome", "user"), + "#class" : erome.EromeUserExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://www.erome.com/search?q=cute", + "#category": ("", "erome", "search"), + "#class" : erome.EromeSearchExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +) diff --git a/test/results/exhentai.py b/test/results/exhentai.py new file mode 100644 index 00000000..293449c8 --- /dev/null +++ b/test/results/exhentai.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import exhentai +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://exhentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#sha1_content": [ + "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", + "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", + ], + + "cost" : int, + "date" : "dt:2018-03-18 20:14:00", + "eh_category" : "Non-H", + "expunged" : False, + "favorites" : r"re:^[12]\d$", + "filecount" : "4", + "filesize" : 1488978, + "gid" : 1200119, + "height" : int, + "image_token" : r"re:[0-9a-f]{10}", + "lang" : "ja", + "language" : "Japanese", + "parent" : "", + "rating" : r"re:\d\.\d+", + "size" : int, + "tags" : [ + "parody:komi-san wa komyushou desu.", + "character:shouko komi", + "group:seventh lowlife", + "other:sample", + ], + "thumb" : "https://exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", + "title" : "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", + "title_jpn" : "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", + "token" : "d55c44d3d0", + "torrentcount": "0", + "uploader" : "klorpa", + "width" : int, +}, + +{ + "#url" : "https://exhentai.org/g/960461/4f0e369d82/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "http://exhentai.org/g/962698/7f02358e00/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://exhentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#count" : 2, +}, + +{ + "#url" : "https://e-hentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#count" : 2, +}, + +{ + "#url" : "https://g.e-hentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, +}, + +{ + "#url" : "https://e-hentai.org/?f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/?f_cats=767&f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/tag/parody:touhou+project", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, + "#pattern" : exhentai.ExhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, + + "gallery_id" : int, + "gallery_token": r"re:^[0-9a-f]{10}$", +}, + +{ + "#url" : "https://e-hentai.org/favorites.php", + "#category": ("", "exhentai", "favorite"), + "#class" : exhentai.ExhentaiFavoriteExtractor, + "#pattern" : r"https?://e-hentai\.org/g/1200119/d55c44d3d0", + "#count" : 1, +}, + +{ + "#url" : "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", + "#category": ("", "exhentai", "favorite"), + "#class" : exhentai.ExhentaiFavoriteExtractor, +}, + +) diff --git a/test/results/fallenangels.py b/test/results/fallenangels.py new file mode 100644 index 00000000..e2167bb4 --- /dev/null +++ b/test/results/fallenangels.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fallenangels + + +__tests__ = ( +{ + "#url" : "https://manga.fascans.com/manga/chronos-ruler/20/1", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "4604a7914566cc2da0ff789aa178e2d1c8c241e3", + "#sha1_metadata": "2dfcc50020e32cd207be88e2a8fac0933e36bdfb", +}, + +{ + "#url" : "http://truyen.fascans.com/manga/hungry-marie/8", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "1f923d9cb337d5e7bbf4323719881794a951c6ae", + "#sha1_metadata": "2bdb7334c0e3eceb9946ffd3132df679b4a94f6a", +}, + +{ + "#url" : "http://manga.fascans.com/manga/rakudai-kishi-no-eiyuutan/19.5", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "273f6863966c83ea79ad5846a2866e08067d3f0e", + "#sha1_metadata": "d1065685bfe0054c4ff2a0f20acb089de4cec253", +}, + +{ + "#url" : "https://manga.fascans.com/manga/chronos-ruler", + "#category": ("", "fallenangels", "manga"), + "#class" : fallenangels.FallenangelsMangaExtractor, + "#sha1_url" : "eea07dd50f5bc4903aa09e2cc3e45c7241c9a9c2", + "#sha1_metadata": "c414249525d4c74ad83498b3c59a813557e59d7e", +}, + +{ + "#url" : "https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", + "#category": ("", "fallenangels", "manga"), + "#class" : fallenangels.FallenangelsMangaExtractor, + "#sha1_url" : "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b", + "#sha1_metadata": "2d2a2a5d9ea5925eb9a47bb13d848967f3af086c", +}, + +) diff --git a/test/results/fanbox.py b/test/results/fanbox.py new file mode 100644 index 00000000..78f7fe54 --- /dev/null +++ b/test/results/fanbox.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fanbox + + +__tests__ = ( +{ + "#url" : "https://xub.fanbox.cc", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, + "#range" : "1-15", + "#count" : ">= 15", + + "creatorId": "xub", + "tags" : list, + "title" : str, +}, + +{ + "#url" : "https://xub.fanbox.cc/posts", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/posts", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/posts/1910054", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#count" : 3, + + "title" : "えま★おうがすと", + "tags" : list, + "hasAdultContent": True, + "isCoverImage" : False, +}, + +{ + "#url" : "https://nekoworks.fanbox.cc/posts/915", + "#comment" : "entry post type, image embedded in html of the post", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#count" : 2, + + "title" : "【SAYORI FAN CLUB】お届け内容", + "tags" : list, + "html" : str, + "hasAdultContent": True, +}, + +{ + "#url" : "https://steelwire.fanbox.cc/posts/285502", + "#comment" : "article post type, imageMap, 2 twitter embeds, fanbox embed", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#options" : {"embeds": True}, + "#count" : 10, + + "title" : "イラスト+SS｜義足の炭鉱少年が義足を見せてくれるだけ 【全体公開版】", + "tags" : list, + "articleBody" : dict, + "hasAdultContent": True, +}, + +{ + "#url" : "https://www.fanbox.cc/@official-en/posts/4326303", + "#comment" : "'content' metadata (#3020)", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + + "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", +}, + +{ + "#url" : "https://mochirong.fanbox.cc/posts/3746116", + "#comment" : "imageMap file order (#2718)", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", +}, + +{ + "#url" : "https://www.pixiv.net/fanbox/creator/52336352", + "#category": ("", "fanbox", "redirect"), + "#class" : fanbox.FanboxRedirectExtractor, + "#pattern" : fanbox.FanboxCreatorExtractor.pattern, +}, + +) diff --git a/test/results/fanleaks.py b/test/results/fanleaks.py new file mode 100644 index 00000000..c420e0ca --- /dev/null +++ b/test/results/fanleaks.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fanleaks +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://fanleaks.club/selti/880", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#pattern" : r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", + + "model_id": "selti", + "model" : "Selti", + "id" : 880, + "type" : "photo", +}, + +{ + "#url" : "https://fanleaks.club/daisy-keech/1038", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#pattern" : r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", + + "model_id": "daisy-keech", + "model" : "Daisy Keech", + "id" : 1038, + "type" : "video", +}, + +{ + "#url" : "https://fanleaks.club/hannahowo/000", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://fanleaks.club/hannahowo", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, + "#pattern" : r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://fanleaks.club/belle-delphine", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, + "#pattern" : r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://fanleaks.club/daisy-keech", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, +}, + +) diff --git a/test/results/fantia.py b/test/results/fantia.py new file mode 100644 index 00000000..706b89d6 --- /dev/null +++ b/test/results/fantia.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fantia + + +__tests__ = ( +{ + "#url" : "https://fantia.jp/fanclubs/6939", + "#category": ("", "fantia", "creator"), + "#class" : fantia.FantiaCreatorExtractor, + "#range" : "1-25", + "#count" : ">= 25", + + "fanclub_user_id": 52152, + "tags" : list, + "title" : str, +}, + +{ + "#url" : "https://fantia.jp/posts/1166373", + "#category": ("", "fantia", "post"), + "#class" : fantia.FantiaPostExtractor, + "#pattern" : r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", + + "blogpost_text" : r"re:^$|This is a test.\n\nThis is a test.\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", + "comment" : """ + +""", + "content_category": r"re:thumb|blog|photo_gallery", + "content_comment" : str, + "content_filename": r"re:|", + "content_title" : r"re:Test (Blog Content \d+|Image Gallery)|thumb", + "date" : "dt:2022-03-09 16:46:12", + "fanclub_id" : 356320, + "fanclub_name" : "Test Fantia", + "fanclub_url" : "https://fantia.jp/fanclubs/356320", + "fanclub_user_id" : 7487131, + "fanclub_user_name": "2022/03/08 15:13:52の名無し", + "file_url" : str, + "filename" : str, + "num" : int, + "plan" : dict, + "post_id" : 1166373, + "post_title" : "Test Fantia Post", + "post_url" : "https://fantia.jp/posts/1166373", + "posted_at" : "Thu, 10 Mar 2022 01:46:12 +0900", + "rating" : "general", + "tags" : [], +}, + +{ + "#url" : "https://fantia.jp/posts/508363", + "#category": ("", "fantia", "post"), + "#class" : fantia.FantiaPostExtractor, + "#count" : 6, + + "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", + "tags" : list, + "rating" : "adult", + "post_id" : 508363, +}, + +) diff --git a/test/results/fapachi.py b/test/results/fapachi.py new file mode 100644 index 00000000..8907e7f8 --- /dev/null +++ b/test/results/fapachi.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fapachi + + +__tests__ = ( +{ + "#url" : "https://fapachi.com/sonson/media/0082", + "#comment" : "NSFW", + "#category": ("", "fapachi", "post"), + "#class" : fapachi.FapachiPostExtractor, + "#pattern" : r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", + + "user": "sonson", + "id" : "0082", +}, + +{ + "#url" : "https://fapachi.com/ferxiita/media/0159", + "#comment" : "NSFW", + "#category": ("", "fapachi", "post"), + "#class" : fapachi.FapachiPostExtractor, +}, + +{ + "#url" : "https://fapachi.com/sonson", + "#category": ("", "fapachi", "user"), + "#class" : fapachi.FapachiUserExtractor, + "#pattern" : fapachi.FapachiPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://fapachi.com/ferxiita/page/3", + "#category": ("", "fapachi", "user"), + "#class" : fapachi.FapachiUserExtractor, +}, + +) diff --git a/test/results/fapello.py b/test/results/fapello.py new file mode 100644 index 00000000..2b8381ca --- /dev/null +++ b/test/results/fapello.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fapello +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://fapello.com/carrykey/530/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#pattern" : r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", + + "model" : "carrykey", + "id" : 530, + "type" : "photo", + "thumbnail": "", +}, + +{ + "#url" : "https://fapello.com/vladislava-661/693/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#pattern" : r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", + + "model" : "vladislava-661", + "id" : 693, + "type" : "video", + "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", +}, + +{ + "#url" : "https://fapello.com/carrykey/000/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://fapello.com/hyoon/", + "#category": ("", "fapello", "model"), + "#class" : fapello.FapelloModelExtractor, + "#pattern" : fapello.FapelloPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://fapello.com/kobaebeefboo/", + "#category": ("", "fapello", "model"), + "#class" : fapello.FapelloModelExtractor, +}, + +{ + "#url" : "https://fapello.com/top-likes/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, + "#pattern" : fapello.FapelloModelExtractor.pattern, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://fapello.com/videos/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, + "#pattern" : fapello.FapelloPostExtractor.pattern, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://fapello.com/top-followers/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/trending/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/popular_videos/twelve_hours/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/popular_videos/week/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +) diff --git a/test/results/fappic.py b/test/results/fappic.py new file mode 100644 index 00000000..b89c4e29 --- /dev/null +++ b/test/results/fappic.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class" : imagehosts.FappicImageExtractor, +}, + +{ + "#url" : "https://www.fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class" : imagehosts.FappicImageExtractor, + "#pattern" : r"https://img\d+\.fappic\.com/img/\w+/test\.png", + "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/fashionnova.py b/test/results/fashionnova.py new file mode 100644 index 00000000..4225264f --- /dev/null +++ b/test/results/fashionnova.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses/?page=1", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses#1", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.fashionnova.com/products/essential-slide-red", + "#category": ("shopify", "fashionnova", "product"), + "#class" : shopify.ShopifyProductExtractor, + "#pattern" : r"https?://cdn\d*\.shopify.com/", + "#count" : 3, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/flats/products/name", + "#category": ("shopify", "fashionnova", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/fireden.py b/test/results/fireden.py new file mode 100644 index 00000000..48549e18 --- /dev/null +++ b/test/results/fireden.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://boards.fireden.net/sci/thread/11264294/", + "#category": ("foolfuuka", "fireden", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", +}, + +{ + "#url" : "https://boards.fireden.net/sci/", + "#category": ("foolfuuka", "fireden", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://boards.fireden.net/_/search/text/test/", + "#category": ("foolfuuka", "fireden", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://boards.fireden.net/sci/gallery/6", + "#category": ("foolfuuka", "fireden", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/flickr.py b/test/results/flickr.py new file mode 100644 index 00000000..5f1fee07 --- /dev/null +++ b/test/results/flickr.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import flickr +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#sha1_content": [ + "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", + "0821a28ee46386e85b02b67cf2720063440a228c", + ], + + "comments" : int, + "description": str, + "extension" : "jpg", + "filename" : "16089302239_de18cd8017_b", + "id" : 16089302239, + "height" : 683, + "label" : "Large", + "media" : "photo", + "url" : str, + "views" : int, + "width" : 1024, +}, + +{ + "#url" : "https://secure.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://m.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://www.flickr.com/photos/145617051@N08/46733161535", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#count" : 1, + + "media": "video", +}, + +{ + "#url" : "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://flic.kr/p/FPVo9U", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://www.flickr.com/photos/zzz/16089302238", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/albums/72157633471741607", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 6, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, + "#pattern" : flickr.FlickrAlbumExtractor.pattern, + "#count" : 2, +}, + +{ + "#url" : "https://secure.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, +}, + +{ + "#url" : "https://m.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, +}, + +{ + "#url" : "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", + "#category": ("", "flickr", "gallery"), + "#class" : flickr.FlickrGalleryExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.flickr.com/groups/bird_headshots/", + "#category": ("", "flickr", "group"), + "#class" : flickr.FlickrGroupExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : "> 150", +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/", + "#category": ("", "flickr", "user"), + "#class" : flickr.FlickrUserExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 28, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/favorites", + "#category": ("", "flickr", "favorite"), + "#class" : flickr.FlickrFavoriteExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 4, +}, + +{ + "#url" : "https://flickr.com/search/?text=mountain", + "#category": ("", "flickr", "search"), + "#class" : flickr.FlickrSearchExtractor, +}, + +{ + "#url" : "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", + "#category": ("", "flickr", "search"), + "#class" : flickr.FlickrSearchExtractor, +}, + +) diff --git a/test/results/foalcon.py b/test/results/foalcon.py new file mode 100644 index 00000000..7639c0df --- /dev/null +++ b/test/results/foalcon.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import szurubooru + + +__tests__ = ( +{ + "#url" : "https://booru.foalcon.com/posts/query=simple_background", + "#category": ("szurubooru", "foalcon", "tag"), + "#class" : szurubooru.SzurubooruTagExtractor, + "#pattern" : r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#range" : "1-150", + "#count" : 150, +}, + +{ + "#url" : "https://booru.foalcon.com/post/30092", + "#category": ("szurubooru", "foalcon", "post"), + "#class" : szurubooru.SzurubooruPostExtractor, + "#pattern" : r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", + "#sha1_url" : "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", + "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", +}, + +) diff --git a/test/results/furaffinity.py b/test/results/furaffinity.py new file mode 100644 index 00000000..187029bb --- /dev/null +++ b/test/results/furaffinity.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import furaffinity + + +__tests__ = ( +{ + "#url" : "https://www.furaffinity.net/gallery/mirlinthloth/", + "#category": ("", "furaffinity", "gallery"), + "#class" : furaffinity.FuraffinityGalleryExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, +}, + +{ + "#url" : "https://www.furaffinity.net/scraps/mirlinthloth/", + "#category": ("", "furaffinity", "scraps"), + "#class" : furaffinity.FuraffinityScrapsExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", + "#count" : ">= 3", +}, + +{ + "#url" : "https://www.furaffinity.net/favorites/mirlinthloth/", + "#category": ("", "furaffinity", "favorite"), + "#class" : furaffinity.FuraffinityFavoriteExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, + + "favorite_id": int, +}, + +{ + "#url" : "https://www.furaffinity.net/search/?q=cute", + "#category": ("", "furaffinity", "search"), + "#class" : furaffinity.FuraffinitySearchExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, +}, + +{ + "#url" : "https://www.furaffinity.net/search/?q=leaf&range=1day", + "#comment" : "first page of search results (#2402)", + "#category": ("", "furaffinity", "search"), + "#class" : furaffinity.FuraffinitySearchExtractor, + "#range" : "1-3", + "#count" : 3, +}, + +{ + "#url" : "https://www.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", + + "artist" : "mirlinthloth", + "artist_url" : "mirlinthloth", + "date" : "dt:2016-11-27 17:24:06", + "description": "A Song made playing the game Cosmic DJ.", + "extension" : "mp3", + "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", + "id" : 21835115, + "tags" : list, + "title" : "Bude's 4 Ever", + "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", + "user" : "mirlinthloth", + "views" : int, + "favorites" : int, + "comments" : int, + "rating" : "General", + "fa_category": "Music", + "theme" : "All", + "species" : "Unspecified / Any", + "gender" : "Any", + "width" : 120, + "height" : 120, +}, + +{ + "#url" : "https://www.furaffinity.net/view/42166511/", + "#comment" : "'external' option (#1492)", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + "#options" : {"external": True}, + "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", + "#count" : 2, +}, + +{ + "#url" : "https://www.furaffinity.net/view/45331225/", + "#comment" : "no tags (#2277)", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + + "artist" : "Kota_Remminders", + "artist_url" : "kotaremminders", + "date" : "dt:2022-01-03 17:49:33", + "fa_category": "Adoptables", + "filename" : "1641232173.kotaremminders_chidopts1", + "gender" : "Any", + "height" : 905, + "id" : 45331225, + "rating" : "General", + "species" : "Unspecified / Any", + "tags" : [], + "theme" : "All", + "title" : "REMINDER", + "width" : 1280, +}, + +{ + "#url" : "https://furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://sfw.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://www.furaffinity.net/full/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class" : furaffinity.FuraffinityUserExtractor, + "#pattern" : "/gallery/mirlinthloth/$", +}, + +{ + "#url" : "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class" : furaffinity.FuraffinityUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "/(gallery|scraps|favorites)/mirlinthloth/$", + "#count" : 3, +}, + +{ + "#url" : "https://www.furaffinity.net/watchlist/by/mirlinthloth/", + "#category": ("", "furaffinity", "following"), + "#class" : furaffinity.FuraffinityFollowingExtractor, + "#pattern" : furaffinity.FuraffinityUserExtractor.pattern, + "#range" : "176-225", + "#count" : 50, +}, + +) diff --git a/test/results/furbooru.py b/test/results/furbooru.py new file mode 100644 index 00000000..682aa005 --- /dev/null +++ b/test/results/furbooru.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://furbooru.org/images/1", + "#category": ("philomena", "furbooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", +}, + +{ + "#url" : "https://furbooru.org/search?q=cute", + "#category": ("philomena", "furbooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://furbooru.org/galleries/27", + "#category": ("philomena", "furbooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#count" : ">= 13", +}, + +) diff --git a/test/results/fuskator.py b/test/results/fuskator.py new file mode 100644 index 00000000..cd68c9c2 --- /dev/null +++ b/test/results/fuskator.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fuskator + + +__tests__ = ( +{ + "#url" : "https://fuskator.com/thumbs/d0GnIzXrSKU/", + "#category": ("", "fuskator", "gallery"), + "#class" : fuskator.FuskatorGalleryExtractor, + "#pattern" : r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", + "#count" : 22, + + "gallery_id" : 473023, + "gallery_hash": "d0GnIzXrSKU", + "title" : r"re:Shaved Brunette Babe Maria Ryabushkina with ", + "views" : int, + "score" : float, + "count" : 22, + "tags" : list, +}, + +{ + "#url" : "https://fuskator.com/expanded/gXpKzjgIidA/index.html", + "#category": ("", "fuskator", "gallery"), + "#class" : fuskator.FuskatorGalleryExtractor, +}, + +{ + "#url" : "https://fuskator.com/search/red_swimsuit/", + "#category": ("", "fuskator", "search"), + "#class" : fuskator.FuskatorSearchExtractor, + "#pattern" : fuskator.FuskatorGalleryExtractor.pattern, + "#count" : ">= 40", +}, + +{ + "#url" : "https://fuskator.com/page/3/swimsuit/quality/", + "#category": ("", "fuskator", "search"), + "#class" : fuskator.FuskatorSearchExtractor, +}, + +) diff --git a/test/results/gelbooru.py b/test/results/gelbooru.py new file mode 100644 index 00000000..8d9ead27 --- /dev/null +++ b/test/results/gelbooru.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru + + +__tests__ = ( +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", + "#category": ("booru", "gelbooru", "tag"), + "#class" : gelbooru.GelbooruTagExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", + "#category": ("booru", "gelbooru", "tag"), + "#class" : gelbooru.GelbooruTagExtractor, + "#pattern" : r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", + "#range" : "196-204", + "#count" : 9, + "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=pool&s=show&id=761", + "#category": ("booru", "gelbooru", "pool"), + "#class" : gelbooru.GelbooruPoolExtractor, + "#count" : 6, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=279415", + "#category": ("booru", "gelbooru", "favorite"), + "#class" : gelbooru.GelbooruFavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#count" : 1, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?s=view&page=post&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&id=313638&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?s=view&id=313638&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?id=313638&page=post&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?id=313638&s=view&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=6018318", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", + + "tags_artist" : "kirisaki_shuusei", + "tags_character": str, + "tags_copyright": "vocaloid", + "tags_general" : str, + "tags_metadata" : str, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5938076", + "#comment" : "video", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#pattern" : r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", + "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5997331", + "#comment" : "notes", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#options" : {"notes": True}, + + "notes": [ + { + "body" : "Look over this way when you talk~", + "height": 553, + "width" : 246, + "x" : 35, + "y" : 72, + }, + { + "body" : """Hey~ +Are you listening~?""", + "height": 557, + "width" : 246, + "x" : 1233, + "y" : 109, + }, + ], +}, + +{ + "#url" : "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", + "#category": ("booru", "gelbooru", "redirect"), + "#class" : gelbooru.GelbooruRedirectExtractor, + "#pattern" : r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", +}, + +) diff --git a/test/results/generic.py b/test/results/generic.py new file mode 100644 index 00000000..dea9f7cd --- /dev/null +++ b/test/results/generic.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import generic + + +__tests__ = ( +{ + "#url" : "https://www.nongnu.org/lzip/", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, + "#count" : 1, + "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", + + "description": "Lossless data compressor", + "imageurl" : "https://www.nongnu.org/lzip/lzip.png", + "keywords" : "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", + "pageurl" : "https://www.nongnu.org/lzip/", +}, + +{ + "#url" : "https://räksmörgås.josefsson.org/", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, + "#pattern" : "^https://räksmörgås.josefsson.org/", + "#count" : 2, +}, + +{ + "#url" : "https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +) diff --git a/test/results/giantessbooru.py b/test/results/giantessbooru.py new file mode 100644 index 00000000..5909c191 --- /dev/null +++ b/test/results/giantessbooru.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://giantessbooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://giantessbooru\.com/index\.php\?q=/image/\d+\.jpg", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://giantessbooru.com/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, +}, + +{ + "#url" : "https://giantessbooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://giantessbooru\.com/index\.php\?q=/image/41\.jpg", + "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", + + "extension": "jpg", + "file_url" : "https://giantessbooru.com/index.php?q=/image/41.jpg", + "filename" : "41", + "height" : 0, + "id" : 41, + "md5" : "", + "size" : 0, + "tags" : "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", + "width" : 1387, +}, + +{ + "#url" : "https://giantessbooru.com/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, +}, + +) diff --git a/test/results/gofile.py b/test/results/gofile.py new file mode 100644 index 00000000..b7d26702 --- /dev/null +++ b/test/results/gofile.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gofile + + +__tests__ = ( +{ + "#url" : "https://gofile.io/d/k6BomI", + "#category": ("", "gofile", "folder"), + "#class" : gofile.GofileFolderExtractor, + "#pattern" : r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", + + "createTime" : int, + "directLink" : r"re:https://store5.gofile.io/download/direct/.+", + "downloadCount": int, + "extension" : "png", + "filename" : "test-テスト-%22&!", + "folder" : { + "childs" : [ + "b0367d79-b8ba-407f-8342-aaf8eb815443", + "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + ], + "code" : "k6BomI", + "createTime" : 1654076165, + "id" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "name" : "root", + "public" : True, + "totalDownloadCount": int, + "totalSize" : 182, + "type" : "folder", + }, + "id" : r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", + "link" : r"re:https://store5.gofile.io/download/.+\.png", + "md5" : r"re:[0-9a-f]{32}", + "mimetype" : "image/png", + "name" : "test-テスト-%22&!.png", + "num" : int, + "parentFolder" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "serverChoosen": "store5", + "size" : 182, + "thumbnail" : r"re:https://store5.gofile.io/download/.+\.png", + "type" : "file", +}, + +{ + "#url" : "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + "#category": ("", "gofile", "folder"), + "#class" : gofile.GofileFolderExtractor, + "#options" : {"website-token": None}, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/hbrowse.py b/test/results/hbrowse.py new file mode 100644 index 00000000..c4b3dcd6 --- /dev/null +++ b/test/results/hbrowse.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hbrowse + + +__tests__ = ( +{ + "#url" : "https://www.hbrowse.com/10363/c00000", + "#category": ("", "hbrowse", "chapter"), + "#class" : hbrowse.HbrowseChapterExtractor, + "#sha1_url" : "6feefbc9f4b98e20d8425ddffa9dd111791dc3e6", + "#sha1_metadata": "274996f6c809e5250b6ff3abbc5147e29f89d9a5", + "#sha1_content" : "44578ebbe176c2c27434966aef22945787e2781e", +}, + +{ + "#url" : "https://www.hbrowse.com/10363", + "#category": ("", "hbrowse", "manga"), + "#class" : hbrowse.HbrowseMangaExtractor, + "#sha1_url" : "b89682bfb86c11d2af0dc47463804ec3ac4aadd6", + "#sha1_metadata": "4b15fda1858a69de1fbf5afddfe47dd893397312", +}, + +) diff --git a/test/results/hentai2read.py b/test/results/hentai2read.py new file mode 100644 index 00000000..01349c2e --- /dev/null +++ b/test/results/hentai2read.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentai2read + + +__tests__ = ( +{ + "#url" : "https://hentai2read.com/amazon_elixir/1/", + "#category": ("", "hentai2read", "chapter"), + "#class" : hentai2read.Hentai2readChapterExtractor, + "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", +}, + +{ + "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", + "#category": ("", "hentai2read", "chapter"), + "#class" : hentai2read.Hentai2readChapterExtractor, + "#pattern" : r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", + "#count" : 36, + + "author" : "Kurisu", + "chapter" : 2, + "chapter_id" : 75152, + "chapter_minor": ".5", + "count" : 36, + "lang" : "en", + "language" : "English", + "manga" : "Popuni Kei Joshi Panic!", + "manga_id" : 13088, + "page" : int, + "title" : "Popuni Kei Joshi Panic! 2.5", + "type" : "Original", +}, + +{ + "#url" : "https://hentai2read.com/amazon_elixir/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#sha1_url" : "273073752d418ec887d7f7211e42b832e8c403ba", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", +}, + +{ + "#url" : "https://hentai2read.com/oshikage_riot/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#sha1_url" : "6595f920a3088a15c2819c502862d45f8eb6bea6", + "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", +}, + +{ + "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#pattern" : hentai2read.Hentai2readChapterExtractor.pattern, + "#range" : "2-3", + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": ".5", + "lang" : "en", + "language" : "English", + "manga" : "Popuni Kei Joshi Panic!", + "manga_id" : 13088, + "title" : str, + "type" : "Original", +}, + +) diff --git a/test/results/hentaicosplays.py b/test/results/hentaicosplays.py new file mode 100644 index 00000000..9760aee8 --- /dev/null +++ b/test/results/hentaicosplays.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaicosplays + + +__tests__ = ( +{ + "#url" : "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.hentai-cosplays.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 18, + "site" : "hentai-cosplays", + "slug" : "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", +}, + +{ + "#url" : "https://fr.porn-images-xxx.com/image/enako-enako-24/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 11, + "site" : "porn-images-xxx", + "title": str, +}, + +{ + "#url" : "https://ja.hentai-img.com/image/hollow-cora-502/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 2, + "site" : "hentai-img", + "title": str, +}, + +) diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py new file mode 100644 index 00000000..4dc4515d --- /dev/null +++ b/test/results/hentaifoundry.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaifoundry +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/profile", + "#category": ("", "hentaifoundry", "user"), + "#class" : hentaifoundry.HentaifoundryUserExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura", + "#category": ("", "hentaifoundry", "pictures"), + "#class" : hentaifoundry.HentaifoundryPicturesExtractor, + "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", + "#category": ("", "hentaifoundry", "pictures"), + "#class" : hentaifoundry.HentaifoundryPicturesExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", + "#category": ("", "hentaifoundry", "scraps"), + "#class" : hentaifoundry.HentaifoundryScrapsExtractor, + "#sha1_url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", + "#category": ("", "hentaifoundry", "scraps"), + "#class" : hentaifoundry.HentaifoundryScrapsExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", + "#category": ("", "hentaifoundry", "favorite"), + "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, + "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", +}, + +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", + "#category": ("", "hentaifoundry", "favorite"), + "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/recent/2018-09-20", + "#category": ("", "hentaifoundry", "recent"), + "#class" : hentaifoundry.HentaifoundryRecentExtractor, + "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range" : "20-30", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/popular", + "#category": ("", "hentaifoundry", "popular"), + "#class" : hentaifoundry.HentaifoundryPopularExtractor, + "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range" : "20-30", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, + "#sha1_url" : "fbf2fd74906738094e2575d2728e8dc3de18a8a3", + "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", + + "artist" : "Tenpura", + "date" : "dt:2016-02-22 14:41:19", + "description": "Thank you!", + "height" : 700, + "index" : 407501, + "media" : "Other digital art", + "ratings" : [ + "Sexual content", + "Contains female nudity", + ], + "score" : int, + "tags" : [ + "collection", + "kancolle", + "kantai", + "shimakaze", + ], + "title" : "shimakaze", + "user" : "Tenpura", + "views" : int, + "width" : 495, +}, + +{ + "#url" : "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, + "#pattern" : "http://pictures.hentai-foundry.com/t/Tenpura/407501/", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, +}, + +{ + "#url" : "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35", + "#category": ("", "hentaifoundry", "stories"), + "#class" : hentaifoundry.HentaifoundryStoriesExtractor, + "#count" : ">= 35", + + "author" : "SnowWolf35", + "chapters" : int, + "comments" : int, + "date" : datetime.datetime, + "description": str, + "index" : int, + "rating" : int, + "ratings" : list, + "status" : r"re:(Inc|C)omplete", + "title" : str, + "user" : "SnowWolf35", + "views" : int, + "words" : int, +}, + +{ + "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", + "#category": ("", "hentaifoundry", "story"), + "#class" : hentaifoundry.HentaifoundryStoryExtractor, + "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", + + "title": "Overwatch High Chapter Voting Location", +}, + +) diff --git a/test/results/hentaifox.py b/test/results/hentaifox.py new file mode 100644 index 00000000..123bd277 --- /dev/null +++ b/test/results/hentaifox.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaifox + + +__tests__ = ( +{ + "#url" : "https://hentaifox.com/gallery/56622/", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + "#pattern" : r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", + "#count" : 24, + "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", +}, + +{ + "#url" : "https://hentaifox.com/gallery/630/", + "#comment" : "'split_tag' element (#1378)", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + + "artist" : [ + "beti", + "betty", + "magi", + "mimikaki", + ], + "characters": [ + "aerith gainsborough", + "tifa lockhart", + "yuffie kisaragi", + ], + "count" : 32, + "gallery_id": 630, + "group" : ["cu-little2"], + "parody" : [ + "darkstalkers | vampire", + "final fantasy vii", + ], + "tags" : [ + "femdom", + "fingering", + "masturbation", + "yuri", + ], + "title" : "Cu-Little Bakanya～", + "type" : "doujinshi", +}, + +{ + "#url" : "https://hentaifox.com/gallery/35261/", + "#comment" : "email-protected title (#4201)", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + + "gallery_id": 35261, + "title" : "ManageM@ster!", + "artist" : ["haritama hiroki"], + "group" : ["studio n.ball"], +}, + +{ + "#url" : "https://hentaifox.com/parody/touhou-project/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/character/reimu-hakurei/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/artist/distance/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/search/touhou/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/group/v-slash/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/tag/heterochromia/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, + "#pattern" : hentaifox.HentaifoxGalleryExtractor.pattern, + "#count" : ">= 60", + + "url" : str, + "gallery_id": int, + "title" : str, +}, + +) diff --git a/test/results/hentaihand.py b/test/results/hentaihand.py new file mode 100644 index 00000000..3d9a34b6 --- /dev/null +++ b/test/results/hentaihand.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaihand + + +__tests__ = ( +{ + "#url" : "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", + "#category": ("", "hentaihand", "gallery"), + "#class" : hentaihand.HentaihandGalleryExtractor, + "#pattern" : r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", + "#count" : 50, + + "artists" : ["Takumi Na Muchi"], + "date" : "dt:2014-06-28 00:00:00", + "gallery_id" : 37387, + "lang" : "en", + "language" : "English", + "parodies" : ["Toaru Kagaku No Railgun"], + "relationships": list, + "tags" : list, + "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", + "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", + "type" : "Doujinshi", +}, + +{ + "#url" : "https://hentaihand.com/en/artist/takumi-na-muchi", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, + "#pattern" : hentaihand.HentaihandGalleryExtractor.pattern, + "#count" : ">= 6", +}, + +{ + "#url" : "https://hentaihand.com/en/tag/full-color", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +{ + "#url" : "https://hentaihand.com/fr/language/japanese", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +{ + "#url" : "https://hentaihand.com/zh/category/manga", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +) diff --git a/test/results/hentaihere.py b/test/results/hentaihere.py new file mode 100644 index 00000000..abe52fe9 --- /dev/null +++ b/test/results/hentaihere.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaihere + + +__tests__ = ( +{ + "#url" : "https://hentaihere.com/m/S13812/1/1/", + "#category": ("", "hentaihere", "chapter"), + "#class" : hentaihere.HentaihereChapterExtractor, + "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", +}, + +{ + "#url" : "https://hentaihere.com/m/S23048/1.5/1/", + "#category": ("", "hentaihere", "chapter"), + "#class" : hentaihere.HentaihereChapterExtractor, + "#pattern" : r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", + "#count" : 32, + + "author" : "Shinozuka Yuuji", + "chapter" : 1, + "chapter_id" : 80186, + "chapter_minor": ".5", + "count" : 32, + "lang" : "en", + "language" : "English", + "manga" : "High School Slut's Love Consultation", + "manga_id" : 23048, + "page" : int, + "title" : "High School Slut's Love Consultation + Girlfriend [Full Color]", + "type" : "Original", +}, + +{ + "#url" : "https://hentaihere.com/m/S13812", + "#category": ("", "hentaihere", "manga"), + "#class" : hentaihere.HentaihereMangaExtractor, + "#sha1_url" : "d1ba6e28bb2162e844f8559c2b2725ba0a093559", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", +}, + +{ + "#url" : "https://hentaihere.com/m/S7608", + "#category": ("", "hentaihere", "manga"), + "#class" : hentaihere.HentaihereMangaExtractor, + "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": "", + "lang" : "en", + "language" : "English", + "manga" : "Oshikake Riot", + "manga_id" : 7608, + "title" : r"re:Oshikake Riot( \d+)?", + "type" : "Original", +}, + +) diff --git a/test/results/hiperdex.py b/test/results/hiperdex.py new file mode 100644 index 00000000..33948f22 --- /dev/null +++ b/test/results/hiperdex.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hiperdex + + +__tests__ = ( +{ + "#url" : "https://hiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, + "#pattern" : r"https://(1st)?hiperdex\d?.(com|net|info)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", + "#count" : 9, + + "artist" : "Sasuga Kei", + "author" : "Sasuga Kei", + "chapter" : 154, + "chapter_minor": ".5", + "description" : r"re:Natsuo Fujii is in love with his teacher, ", + "genre" : list, + "manga" : "Domestic na Kanojo", + "release" : 2014, + "score" : float, + "type" : "Manga", +}, + +{ + "#url" : "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.com/manga/1603231576-youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, + "#pattern" : hiperdex.HiperdexChapterExtractor.pattern, + "#count" : 51, + + "artist" : "Bolp", + "author" : "Abyo4", + "chapter" : int, + "chapter_minor": "", + "description" : r"re:I didn’t think much of the creepy girl in ", + "genre" : list, + "manga" : "You’re Not That Special!", + "release" : 2019, + "score" : float, + "status" : "Completed", + "type" : "Manhwa", +}, + +{ + "#url" : "https://hiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://1sthiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://1sthiperdex.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.com/manga-author/viagra/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, + "#pattern" : hiperdex.HiperdexMangaExtractor.pattern, + "#count" : ">= 6", +}, + +) diff --git a/test/results/hitomi.py b/test/results/hitomi.py new file mode 100644 index 00000000..10e94135 --- /dev/null +++ b/test/results/hitomi.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hitomi + + +__tests__ = ( +{ + "#url" : "https://hitomi.la/galleries/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#pattern" : r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", + "#count" : 16, + "#sha1_metadata": "86af5371f38117a07407f11af689bdd460b09710", +}, + +{ + "#url" : "https://hitomi.la/galleries/1401410.html", + "#comment" : "download test", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#range" : "1", + "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", +}, + +{ + "#url" : "https://hitomi.la/galleries/733697.html", + "#comment" : "Game CG with scenes (#321)", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#count" : 210, +}, + +{ + "#url" : "https://hitomi.la/galleries/1045954.html", + "#comment" : "fallback for galleries only available through /reader/ URLs", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#count" : 1413, +}, + +{ + "#url" : "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", + "#comment" : "gallery with 'broken' redirect", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#options" : {"format": "original"}, + "#pattern" : r"https://[a-c]b\.hitomi\.la/images/\d+/\d+/[0-9a-f]{64}\.jpg", + "#count" : 10, +}, + +{ + "#url" : "https://hitomi.la/cg/1615823.html", + "#comment" : "no tags", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#options" : {"format": "avif"}, + "#pattern" : r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", + "#count" : 22, +}, + +{ + "#url" : "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/manga/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/doujinshi/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/cg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/gamecg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/reader/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/tag/screenshots-japanese.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, + "#pattern" : hitomi.HitomiGalleryExtractor.pattern, + "#count" : ">= 35", +}, + +{ + "#url" : "https://hitomi.la/artist/a1-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/group/initial%2Dg-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/series/amnesia-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/type/doujinshi-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/character/a2-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +) diff --git a/test/results/horne.py b/test/results/horne.py new file mode 100644 index 00000000..eb7b0a79 --- /dev/null +++ b/test/results/horne.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nijie +import datetime + + +__tests__ = ( +{ + "#url" : "https://horne.red/members.php?id=58000", + "#category": ("Nijie", "horne", "user"), + "#class" : nijie.NijieUserExtractor, +}, + +{ + "#url" : "https://horne.red/members_illust.php?id=58000", + "#category": ("Nijie", "horne", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", + "#range" : "1-20", + "#count" : 20, + + "artist_id" : 58000, + "artist_name": "のえるわ", + "date" : datetime.datetime, + "description": str, + "image_id" : int, + "num" : int, + "tags" : list, + "title" : str, + "url" : str, + "user_id" : 58000, + "user_name" : "のえるわ", +}, + +{ + "#url" : "https://horne.red/members_dojin.php?id=58000", + "#category": ("Nijie", "horne", "doujin"), + "#class" : nijie.NijieDoujinExtractor, +}, + +{ + "#url" : "https://horne.red/user_like_illust_view.php?id=58000", + "#category": ("Nijie", "horne", "favorite"), + "#class" : nijie.NijieFavoriteExtractor, + "#range" : "1-5", + "#count" : 5, + + "user_id" : 58000, + "user_name": "のえるわ", +}, + +{ + "#url" : "https://horne.red/history_nuita.php?id=58000", + "#category": ("Nijie", "horne", "nuita"), + "#class" : nijie.NijieNuitaExtractor, +}, + +{ + "#url" : "https://horne.red/like_user_view.php", + "#category": ("Nijie", "horne", "feed"), + "#class" : nijie.NijieFeedExtractor, +}, + +{ + "#url" : "https://horne.red/like_my.php", + "#category": ("Nijie", "horne", "followed"), + "#class" : nijie.NijiefollowedExtractor, +}, + +{ + "#url" : "https://horne.red/view.php?id=8716", + "#category": ("Nijie", "horne", "image"), + "#class" : nijie.NijieImageExtractor, + "#count" : 4, + + "artist_id" : 58000, + "artist_name": "のえるわ", + "date" : "dt:2018-02-04 14:47:24", + "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", + "image_id" : 8716, + "tags" : [ + "男の娘", + "フェラ", + "オリキャラ", + "うちのこ", + ], + "title" : "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", + "user_id" : 58000, + "user_name" : "のえるわ", +}, + +) diff --git a/test/results/hotleak.py b/test/results/hotleak.py new file mode 100644 index 00000000..4e577d11 --- /dev/null +++ b/test/results/hotleak.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hotleak +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://hotleak.vip/kaiyakawaii/photo/1617145", + "#category": ("", "hotleak", "post"), + "#class" : hotleak.HotleakPostExtractor, + "#pattern" : r"https://hotleak\.vip/storage/images/3625/1617145/fefdd5988dfcf6b98cc9e11616018868\.jpg", + + "id" : 1617145, + "creator" : "kaiyakawaii", + "type" : "photo", + "filename" : "fefdd5988dfcf6b98cc9e11616018868", + "extension": "jpg", +}, + +{ + "#url" : "https://hotleak.vip/lilmochidoll/video/1625538", + "#category": ("", "hotleak", "post"), + "#class" : hotleak.HotleakPostExtractor, + "#pattern" : r"ytdl:https://cdn8-leak\.camhdxx\.com/1661/1625538/index\.m3u8", + + "id" : 1625538, + "creator" : "lilmochidoll", + "type" : "video", + "filename" : "index", + "extension": "mp4", +}, + +{ + "#url" : "https://hotleak.vip/kaiyakawaii", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#range" : "1-200", + "#count" : 200, +}, + +{ + "#url" : "https://hotleak.vip/stellaviolet", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#count" : "> 600", +}, + +{ + "#url" : "https://hotleak.vip/doesnotexist", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://hotleak.vip/photos", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, + "#pattern" : hotleak.HotleakPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://hotleak.vip/videos", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, +}, + +{ + "#url" : "https://hotleak.vip/creators", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, + "#pattern" : hotleak.HotleakCreatorExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://hotleak.vip/hot", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, +}, + +{ + "#url" : "https://hotleak.vip/search?search=gallery-dl", + "#category": ("", "hotleak", "search"), + "#class" : hotleak.HotleakSearchExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://hotleak.vip/search?search=hannah", + "#category": ("", "hotleak", "search"), + "#class" : hotleak.HotleakSearchExtractor, + "#count" : "> 30", +}, + +) diff --git a/test/results/hypnohub.py b/test/results/hypnohub.py new file mode 100644 index 00000000..d979b620 --- /dev/null +++ b/test/results/hypnohub.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", + "#category": ("gelbooru_v02", "hypnohub", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=pool&s=show&id=61", + "#category": ("gelbooru_v02", "hypnohub", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, + "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", + "#category": ("gelbooru_v02", "hypnohub", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=post&s=view&id=1439", + "#category": ("gelbooru_v02", "hypnohub", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : { + "tags" : True, + "notes": True, + }, + "#pattern" : r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", + "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", + + "tags_artist" : "brokenteapot", + "tags_character": "hsien-ko", + "tags_copyright": "capcom darkstalkers", + "tags_general" : str, + "tags_metadata" : "dialogue text translated", + "notes" : [ + { + "body" : "Master Master Master Master Master Master", + "height": 83, + "id" : 10577, + "width" : 129, + "x" : 259, + "y" : 20, + }, + { + "body" : "Response Response Response Response Response Response", + "height": 86, + "id" : 10578, + "width" : 125, + "x" : 126, + "y" : 20, + }, + { + "body" : "Obedience Obedience Obedience Obedience Obedience Obedience", + "height": 80, + "id" : 10579, + "width" : 98, + "x" : 20, + "y" : 20, + }, + ], +}, + +) diff --git a/test/results/idolcomplex.py b/test/results/idolcomplex.py new file mode 100644 index 00000000..6aced68c --- /dev/null +++ b/test/results/idolcomplex.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import idolcomplex + + +__tests__ = ( +{ + "#url" : "https://idol.sankakucomplex.com/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, + "#pattern" : r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", + "#range" : "18-22", + "#count" : 5, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/?tags=order:favcount", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, + "#range" : "18-22", + "#count" : 5, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/pool/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class" : idolcomplex.IdolcomplexPoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/post/show/694215", + "#category": ("booru", "idolcomplex", "post"), + "#class" : idolcomplex.IdolcomplexPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_idol" : str, + "tags_medium" : str, + "tags_general" : str, +}, + +) diff --git a/test/results/illusioncardsbooru.py b/test/results/illusioncardsbooru.py new file mode 100644 index 00000000..ba079364 --- /dev/null +++ b/test/results/illusioncardsbooru.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", + "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", + "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", + "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", + "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", +}, + +) diff --git a/test/results/imagebam.py b/test/results/imagebam.py new file mode 100644 index 00000000..759e2dc4 --- /dev/null +++ b/test/results/imagebam.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagebam +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#sha1_url" : "76d976788ae2757ac81694736b07b72356f5c4c8", + "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", + "#sha1_content" : "596e6bfa157f2c7169805d50075c2986549973a8", +}, + +{ + "#url" : "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#count" : 107, + "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", +}, + +{ + "#url" : "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://www.imagebam.com/view/GA3MT1", + "#comment" : "/view/ path (#2378)", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#sha1_url" : "35018ce1e00a2d2825a33d3cd37857edaf804919", + "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", +}, + +{ + "#url" : "https://www.imagebam.com/image/94d56c502511890", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url" : "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", + "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, +}, + +{ + "#url" : "https://www.imagebam.com/image/0850951366904951", + "#comment" : "NSFW (#1534)", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", +}, + +{ + "#url" : "https://www.imagebam.com/view/ME8JOQP", + "#comment" : "/view/ path (#2378)", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url" : "4dca72bbe61a0360185cf4ab2bed8265b49565b8", + "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", + "#sha1_content" : "f81008666b17a42d8834c4749b910e1dc10a6e83", +}, + +) diff --git a/test/results/imagechest.py b/test/results/imagechest.py new file mode 100644 index 00000000..4b6a2ba3 --- /dev/null +++ b/test/results/imagechest.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagechest +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imgchest.com/p/3na7kr3by8d", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count" : 3, + "#sha1_url" : "7328ca4ec2459378d725e3be19f661d2b045feda", + "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", + + "count" : 3, + "gallery_id": "3na7kr3by8d", + "num" : int, + "title" : "Wizardry - Video Game From The Mid 80's", +}, + +{ + "#url" : "https://imgchest.com/p/9p4n3q2z7nq", + "#comment" : "'Load More Files' button (#4028)", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count" : 52, + "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", +}, + +{ + "#url" : "https://imgchest.com/p/xxxxxxxxxxx", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/imagefap.py b/test/results/imagefap.py new file mode 100644 index 00000000..99a22166 --- /dev/null +++ b/test/results/imagefap.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagefap + + +__tests__ = ( +{ + "#url" : "https://www.imagefap.com/gallery/7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#sha1_metadata": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b", + "#sha1_content" : "694a0a57385980a6f90fbc296cadcd6c11ba2dab", +}, + +{ + "#url" : "https://www.imagefap.com/gallery/7876223", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#count" : 44, + + "categories" : [ + "Asses", + "Softcore", + "Pornstars", + ], + "count" : 44, + "description": "", + "gallery_id" : 7876223, + "image_id" : int, + "num" : int, + "tags" : [ + "big ass", + "panties", + "horny", + "pussy", + "exposed", + "outdoor", + ], + "title" : "Kelsi Monroe in lingerie", + "uploader" : "BdRachel", +}, + +{ + "#url" : "https://www.imagefap.com/gallery/6180555", + "#comment" : "description (#3905)", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#range" : "1", + + "categories" : [ + "Amateur", + "Softcore", + "Homemade", + ], + "count" : 36, + "description": "Nude and dressed sluts showing off the goods", + "gallery_id" : 6180555, + "image_id" : int, + "num" : int, + "tags" : [], + "title" : "Dressed or Undressed MG*", + "uploader" : "splitopen", +}, + +{ + "#url" : "https://www.imagefap.com/pictures/7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://beta.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class" : imagefap.ImagefapImageExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/65/196/1962981893\.jpg", + + "date" : "21/08/2014", + "gallery_id": 7876223, + "height" : 1600, + "image_id" : 1962981893, + "title" : "Kelsi Monroe in lingerie", + "uploader" : "BdRachel", + "width" : 1066, +}, + +{ + "#url" : "https://beta.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class" : imagefap.ImagefapImageExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/organizer/409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : r"https://www\.imagefap\.com/gallery/7876223", + "#count" : 1, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, + "#pattern" : imagefap.ImagefapFolderExtractor.pattern, + "#count" : ">= 18", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1862791", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, + "#pattern" : r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", + "#count" : 1, +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel/galleries", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +{ + "#url" : "https://beta.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +) diff --git a/test/results/imagetwist.py b/test/results/imagetwist.py new file mode 100644 index 00000000..4606a773 --- /dev/null +++ b/test/results/imagetwist.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, + "#sha1_url" : "8d5e168c0bee30211f821c6f3b2116e419d42671", + "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +) diff --git a/test/results/imagevenue.py b/test/results/imagevenue.py new file mode 100644 index 00000000..f57b1c94 --- /dev/null +++ b/test/results/imagevenue.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://www.imagevenue.com/ME13LS07", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#pattern" : r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", + "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", + "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", +}, + +{ + "#url" : "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", +}, + +{ + "#url" : "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", +}, + +) diff --git a/test/results/imgbb.py b/test/results/imgbb.py new file mode 100644 index 00000000..adc1dcf9 --- /dev/null +++ b/test/results/imgbb.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgbb +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://ibb.co/album/i5PggF", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#range" : "1-80", + "#sha1_url" : "70afec9fcc3a6de62a6b644b487d892d8d47cf1a", + "#sha1_metadata": "569e1d88ebdd27655387559cdf1cd526a3e1ab69", +}, + +{ + "#url" : "https://ibb.co/album/i5PggF?sort=title_asc", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#range" : "1-80", + "#sha1_url" : "afdf5fc95d8e09d77e8f44312f3e9b843987bb5a", + "#sha1_metadata": "f090e14d0e5f7868595082b2c95da1309c84872d", +}, + +{ + "#url" : "https://ibb.co/album/kYKpwF", + "#comment" : "no user data (#471)", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", + + "user": "", +}, + +{ + "#url" : "https://ibb.co/album/hqgWrF", + "#comment" : "private", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://folkie.imgbb.com", + "#category": ("", "imgbb", "user"), + "#class" : imgbb.ImgbbUserExtractor, + "#pattern" : r"https?://i\.ibb\.co/\w+/[^/?#]+", + "#range" : "1-80", +}, + +{ + "#url" : "https://ibb.co/fUqh5b", + "#category": ("", "imgbb", "image"), + "#class" : imgbb.ImgbbImageExtractor, + "#pattern" : r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", + "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", + + "id" : "fUqh5b", + "title" : "Arundel Ireeman 5", + "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", + "width" : 960, + "height" : 719, + "user" : "folkie", + "extension": "jpg", +}, + +) diff --git a/test/results/imgbox.py b/test/results/imgbox.py new file mode 100644 index 00000000..ed0be593 --- /dev/null +++ b/test/results/imgbox.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgbox +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imgbox.com/g/JaX5V5HX7g", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#sha1_url" : "da4f15b161461119ee78841d4b8e8d054d95f906", + "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", + "#sha1_content" : "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", +}, + +{ + "#url" : "https://imgbox.com/g/cUGEkRbdZZ", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#sha1_url" : "76506a3aab175c456910851f66227e90484ca9f7", + "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", +}, + +{ + "#url" : "https://imgbox.com/g/JaX5V5HX7h", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://imgbox.com/qHhw7lpG", + "#category": ("", "imgbox", "image"), + "#class" : imgbox.ImgboxImageExtractor, + "#sha1_url" : "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", + "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://imgbox.com/qHhw7lpH", + "#category": ("", "imgbox", "image"), + "#class" : imgbox.ImgboxImageExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/imgclick.py b/test/results/imgclick.py new file mode 100644 index 00000000..983f21f4 --- /dev/null +++ b/test/results/imgclick.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", + "#category": ("imagehost", "imgclick", "image"), + "#class" : imagehosts.ImgclickImageExtractor, + "#sha1_url" : "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", + "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/imgspice.py b/test/results/imgspice.py new file mode 100644 index 00000000..b2691b6a --- /dev/null +++ b/test/results/imgspice.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://imgspice.com/nwfwtpyog50y/test.png.html", + "#category": ("imagehost", "imgspice", "image"), + "#class" : imagehosts.ImgspiceImageExtractor, + "#sha1_url" : "b8c30a8f51ee1012959a4cfd46197fabf14de984", + "#sha1_metadata": "100e310a19a2fa22d87e1bbc427ecb9f6501e0c0", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/imgth.py b/test/results/imgth.py new file mode 100644 index 00000000..333e1352 --- /dev/null +++ b/test/results/imgth.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgth + + +__tests__ = ( +{ + "#url" : "https://imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class" : imgth.ImgthGalleryExtractor, + "#pattern" : r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", + "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", + + "count" : 12, + "date" : "dt:2009-11-25 18:21:00", + "extension" : "jpg", + "filename" : r"re:wallpaper-anime_\w+", + "gallery_id": 37, + "num" : int, + "title" : "Wallpaper anime", + "user" : "celebrities", +}, + +{ + "#url" : "https://www.imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class" : imgth.ImgthGalleryExtractor, +}, + +) diff --git a/test/results/imgur.py b/test/results/imgur.py new file mode 100644 index 00000000..4fedc600 --- /dev/null +++ b/test/results/imgur.py @@ -0,0 +1,367 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgur +from gallery_dl import exception +import datetime + + +__tests__ = ( +{ + "#url" : "https://imgur.com/21yMxCS", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#sha1_url" : "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "account_id" : 0, + "comment_count" : int, + "cover_id" : "21yMxCS", + "date" : "dt:2016-11-10 14:24:35", + "description" : "", + "downvote_count": int, + "duration" : 0, + "ext" : "png", + "favorite" : False, + "favorite_count": 0, + "has_sound" : False, + "height" : 32, + "id" : "21yMxCS", + "image_count" : 1, + "in_most_viral" : False, + "is_ad" : False, + "is_album" : False, + "is_animated" : False, + "is_looping" : False, + "is_mature" : False, + "is_pending" : False, + "mime_type" : "image/png", + "name" : "test-テスト", + "point_count" : int, + "privacy" : "", + "score" : int, + "size" : 182, + "title" : "Test", + "upvote_count" : int, + "url" : "https://i.imgur.com/21yMxCS.png", + "view_count" : int, + "width" : 64, +}, + +{ + "#url" : "http://imgur.com/0gybAXR", + "#comment" : "gifv/mp4 video", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#sha1_url" : "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", + "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", +}, + +{ + "#url" : "https://imgur.com/XFfsmuC", + "#comment" : "missing title in API response (#467)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + + "title": "Tears are a natural response to irritants", +}, + +{ + "#url" : "https://imgur.com/1Nily2P", + "#comment" : "animated png", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#pattern" : "https://i.imgur.com/1Nily2P.png", +}, + +{ + "#url" : "https://imgur.com/zzzzzzz", + "#comment" : "not found", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://m.imgur.com/r/Celebs/iHJ7tsM", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://www.imgur.com/21yMxCS", + "#comment" : "www", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://m.imgur.com/21yMxCS", + "#comment" : "mobile", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.com/zxaY6", + "#comment" : "5 character key", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.io/zxaY6", + "#comment" : ".io", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/21yMxCS.png", + "#comment" : "direct link", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.io/21yMxCS.png", + "#comment" : "direct link .io", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/21yMxCSh.png", + "#comment" : "direct link thumbnail", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/zxaY6.gif", + "#comment" : "direct link (short)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/zxaY6s.gif", + "#comment" : "direct link (short; thumb)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.com/a/TcBmP", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", + + "album" : { + "account_id" : 0, + "comment_count" : int, + "cover_id" : "693j2Kr", + "date" : "dt:2015-10-09 10:37:50", + "description" : "", + "downvote_count": 0, + "favorite" : False, + "favorite_count": 0, + "id" : "TcBmP", + "image_count" : 19, + "in_most_viral" : False, + "is_ad" : False, + "is_album" : True, + "is_mature" : False, + "is_pending" : False, + "privacy" : "private", + "score" : int, + "title" : "138", + "upvote_count" : int, + "url" : "https://imgur.com/a/TcBmP", + "view_count" : int, + "virality" : int, + }, + "account_id" : 0, + "count" : 19, + "date" : datetime.datetime, + "description": "", + "ext" : "jpg", + "has_sound" : False, + "height" : int, + "id" : str, + "is_animated": False, + "is_looping" : False, + "mime_type" : "image/jpeg", + "name" : str, + "num" : int, + "size" : int, + "title" : str, + "type" : "image", + "updated_at" : None, + "url" : str, + "width" : int, +}, + +{ + "#url" : "https://imgur.com/a/eD9CT", + "#comment" : "large album", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "de748c181a04d18bef1de9d4f4866ef0a06d632b", +}, + +{ + "#url" : "https://imgur.com/a/RhJXhVT/all", + "#comment" : "7 character album hash", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", +}, + +{ + "#url" : "https://imgur.com/a/TcBmQ", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://imgur.com/a/pjOnJA0", + "#comment" : "empty, no 'media' (#2557)", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.imgur.com/a/TcBmP", + "#comment" : "www", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://imgur.io/a/TcBmP", + "#comment" : ".io", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://m.imgur.com/a/TcBmP", + "#comment" : "mobile", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://imgur.com/gallery/zf2fIms", + "#comment" : "non-album gallery (#380)", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, + "#pattern" : "https://imgur.com/zf2fIms", +}, + +{ + "#url" : "https://imgur.com/gallery/eD9CT", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, + "#pattern" : "https://imgur.com/a/eD9CT", +}, + +{ + "#url" : "https://imgur.com/t/unmuted/26sEhNr", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.com/t/cat/qSB8NbN", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.io/t/cat/qSB8NbN", + "#comment" : ".io", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/posts", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/submitted", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/favorites", + "#category": ("", "imgur", "favorite"), + "#class" : imgur.ImgurFavoriteExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896757/public", + "#category": ("", "imgur", "favorite-folder"), + "#class" : imgur.ImgurFavoriteFolderExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#count" : 3, +}, + +{ + "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896741/private", + "#category": ("", "imgur", "favorite-folder"), + "#class" : imgur.ImgurFavoriteFolderExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#count" : 5, +}, + +{ + "#url" : "https://imgur.com/r/pics", + "#category": ("", "imgur", "subreddit"), + "#class" : imgur.ImgurSubredditExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/t/animals", + "#category": ("", "imgur", "tag"), + "#class" : imgur.ImgurTagExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/search?q=cute+cat", + "#category": ("", "imgur", "search"), + "#class" : imgur.ImgurSearchExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +) diff --git a/test/results/imxto.py b/test/results/imxto.py new file mode 100644 index 00000000..430b388b --- /dev/null +++ b/test/results/imxto.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imx.to/i/1qdeva", + "#comment" : "new-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url" : "ab2173088a6cdef631d7a47dec4a5da1c6a00130", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "size" : 18, + "width" : 64, + "height": 32, + "hash" : "94d56c599223c59f3feb71ea603484d1", +}, + +{ + "#url" : "https://imx.to/img-57a2050547b97.html", + "#comment" : "old-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url" : "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", + + "size" : 5284, + "width" : 320, + "height": 160, + "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", +}, + +{ + "#url" : "https://img.yt/img-57a2050547b97.html", + "#comment" : "img.yt domain", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", +}, + +{ + "#url" : "https://imx.to/img-57a2050547b98.html", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://imx.to/g/ozdy", + "#category": ("imagehost", "imxto", "gallery"), + "#class" : imagehosts.ImxtoGalleryExtractor, + "#pattern" : imagehosts.ImxtoImageExtractor.pattern, + "#count" : 40, + + "title": "untitled gallery", +}, + +) diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py new file mode 100644 index 00000000..7129877e --- /dev/null +++ b/test/results/inkbunny.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import inkbunny +import datetime + + +__tests__ = ( +{ + "#url" : "https://inkbunny.net/soina", + "#category": ("", "inkbunny", "user"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", + "#range" : "20-50", + + "date" : datetime.datetime, + "deleted" : bool, + "file_id" : r"re:[0-9]+", + "filename" : r"re:[0-9]+_soina_\w+", + "full_file_md5" : r"re:[0-9a-f]{32}", + "mimetype" : str, + "submission_id" : r"re:[0-9]+", + "user_id" : "20969", + "comments_count" : r"re:[0-9]+", + "favorite" : bool, + "favorites_count": r"re:[0-9]+", + "friends_only" : bool, + "guest_block" : bool, + "hidden" : bool, + "pagecount" : r"re:[0-9]+", + "pools" : list, + "pools_count" : int, + "public" : bool, + "rating_id" : r"re:[0-9]+", + "rating_name" : str, + "ratings" : list, + "scraps" : bool, + "tags" : list, + "title" : str, + "type_name" : str, + "username" : "soina", + "views" : str, +}, + +{ + "#url" : "https://inkbunny.net/gallery/soina", + "#category": ("", "inkbunny", "gallery"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#range" : "1-25", + + "scraps": False, +}, + +{ + "#url" : "https://inkbunny.net/scraps/soina", + "#category": ("", "inkbunny", "scraps"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#range" : "1-25", + + "scraps": True, +}, + +{ + "#url" : "https://inkbunny.net/poolview_process.php?pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class" : inkbunny.InkbunnyPoolExtractor, + "#count" : 9, + + "pool_id": "28985", +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", + "#category": ("", "inkbunny", "pool"), + "#class" : inkbunny.InkbunnyPoolExtractor, +}, + +{ + "#url" : "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class" : inkbunny.InkbunnyFavoriteExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", + "#range" : "20-50", + + "favs_user_id": "20969", +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class" : inkbunny.InkbunnyFavoriteExtractor, +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", + "#category": ("", "inkbunny", "search"), + "#class" : inkbunny.InkbunnySearchExtractor, + "#range" : "1-10", + "#count" : 10, + + "search": { + "rid" : "ffffffffff", + "mode" : "search", + "page" : "1", + "orderby" : "create_datetime", + "text" : "cute", + "stringtype" : "and", + "keywords" : "yes", + "title" : "yes", + "description": "no", + }, +}, + +{ + "#url" : "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class" : inkbunny.InkbunnyFollowingExtractor, + "#pattern" : inkbunny.InkbunnyUserExtractor.pattern, + "#count" : ">= 90", +}, + +{ + "#url" : "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", + "#category": ("", "inkbunny", "following"), + "#class" : inkbunny.InkbunnyFollowingExtractor, +}, + +{ + "#url" : "https://inkbunny.net/s/1829715", + "#category": ("", "inkbunny", "post"), + "#class" : inkbunny.InkbunnyPostExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", + "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", +}, + +{ + "#url" : "https://inkbunny.net/s/2044094", + "#category": ("", "inkbunny", "post"), + "#class" : inkbunny.InkbunnyPostExtractor, + "#count" : 4, +}, + +) diff --git a/test/results/instagram.py b/test/results/instagram.py new file mode 100644 index 00000000..7f601426 --- /dev/null +++ b/test/results/instagram.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import instagram + + +__tests__ = ( +{ + "#url" : "https://www.instagram.com/instagram/", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/?hl=en", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/id:25025320/", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/posts/", + "#category": ("", "instagram", "posts"), + "#class" : instagram.InstagramPostsExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/reels/", + "#category": ("", "instagram", "reels"), + "#class" : instagram.InstagramReelsExtractor, + "#range" : "40-60", + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.instagram.com/instagram/tagged/", + "#category": ("", "instagram", "tagged"), + "#class" : instagram.InstagramTaggedExtractor, + "#range" : "1-16", + "#count" : ">= 16", + + "tagged_owner_id" : "25025320", + "tagged_username" : "instagram", + "tagged_full_name": "Instagram", +}, + +{ + "#url" : "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", + "#category": ("", "instagram", "guide"), + "#class" : instagram.InstagramGuideExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/", + "#category": ("", "instagram", "saved"), + "#class" : instagram.InstagramSavedExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/all-posts/", + "#category": ("", "instagram", "saved"), + "#class" : instagram.InstagramSavedExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/collection_name/123456789/", + "#category": ("", "instagram", "collection"), + "#class" : instagram.InstagramCollectionExtractor, +}, + +{ + "#url" : "https://www.instagram.com/stories/instagram/", + "#category": ("", "instagram", "stories"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/stories/highlights/18042509488170095/", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://instagram.com/stories/geekmig/2724343156064789461", + "#category": ("", "instagram", "stories"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/highlights", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramHighlightsExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/following", + "#category": ("", "instagram", "following"), + "#class" : instagram.InstagramFollowingExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/explore/tags/instagram/", + "#category": ("", "instagram", "tag"), + "#class" : instagram.InstagramTagExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/avatar", + "#category": ("", "instagram", "avatar"), + "#class" : instagram.InstagramAvatarExtractor, + "#pattern" : r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", +}, + +{ + "#url" : "https://www.instagram.com/p/BqvsDleB3lV/", + "#comment" : "GraphImage", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", + + "date" : "dt:2018-11-29 01:04:04", + "description" : str, + "height" : int, + "likes" : int, + "location_id" : "214424288", + "location_slug" : "hong-kong", + "location_url" : r"re:/explore/locations/214424288/hong-kong/", + "media_id" : "1922949326347663701", + "shortcode" : "BqvsDleB3lV", + "post_id" : "1922949326347663701", + "post_shortcode": "BqvsDleB3lV", + "post_url" : "https://www.instagram.com/p/BqvsDleB3lV/", + "tags" : ["#WHPsquares"], + "typename" : "GraphImage", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/p/BoHk1haB5tM/", + "#comment" : "GraphSidecar", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#count" : 5, + + "sidecar_media_id": "1875629777499953996", + "sidecar_shortcode": "BoHk1haB5tM", + "post_id" : "1875629777499953996", + "post_shortcode" : "BoHk1haB5tM", + "post_url" : "https://www.instagram.com/p/BoHk1haB5tM/", + "num" : int, + "likes" : int, + "username" : "instagram", +}, + +{ + "#url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", + "#comment" : "GraphVideo", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"/46840863_726311431074534_7805566102611403091_n\.mp4", + + "date" : "dt:2018-11-29 19:23:58", + "description": str, + "height" : int, + "likes" : int, + "media_id" : "1923502432034620000", + "post_url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", + "shortcode" : "Bqxp0VSBgJg", + "tags" : ["#ASMR"], + "typename" : "GraphVideo", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/tv/BkQjCfsBIzi/", + "#comment" : "GraphVideo (IGTV)", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"/10000000_597132547321814_702169244961988209_n\.mp4", + + "date" : "dt:2018-06-20 19:51:32", + "description": str, + "height" : int, + "likes" : int, + "media_id" : "1806097553666903266", + "post_url" : "https://www.instagram.com/p/BkQjCfsBIzi/", + "shortcode" : "BkQjCfsBIzi", + "typename" : "GraphVideo", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/p/BtOvDOfhvRr/", + "#comment" : "GraphSidecar with 2 embedded GraphVideo objects", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#count" : 2, + + "post_url" : "https://www.instagram.com/p/BtOvDOfhvRr/", + "sidecar_media_id": "1967717017113261163", + "sidecar_shortcode": "BtOvDOfhvRr", + "video_url" : str, +}, + +{ + "#url" : "https://www.instagram.com/p/B_2lf3qAd3y/", + "#comment" : "GraphImage with tagged user", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + + "tagged_users": [{ + "id" : "1246468638", + "username" : "kaaymbl", + "full_name": "Call Me Kay", +}], +}, + +{ + "#url" : "https://www.instagram.com/dm/p/CW042g7B9CY/", + "#comment" : "URL with username (#2085)", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, +}, + +{ + "#url" : "https://www.instagram.com/reel/CDg_6Y1pxWu/", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, +}, + +) diff --git a/test/results/issuu.py b/test/results/issuu.py new file mode 100644 index 00000000..5e086d1e --- /dev/null +++ b/test/results/issuu.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import issuu + + +__tests__ = ( +{ + "#url" : "https://issuu.com/issuu/docs/motions-1-2019/", + "#category": ("", "issuu", "publication"), + "#class" : issuu.IssuuPublicationExtractor, + "#pattern" : r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", + "#count" : 36, + + "document" : { + "access" : "PUBLIC", + "contentRating": { + "isAdsafe" : True, + "isExplicit": False, + "isReviewed": True, + }, + "date" : "dt:2019-09-16 00:00:00", + "description" : r"re:Motions, the brand new publication by I", + "documentName" : "motions-1-2019", + "downloadable" : False, + "pageCount" : 36, + "publicationId": "d99ec95935f15091b040cb8060f05510", + "title" : "Motions by Issuu - Issue 1", + "username" : "issuu", + }, + "extension": "jpg", + "filename" : r"re:page_\d+", + "num" : int, +}, + +{ + "#url" : "https://issuu.com/issuu", + "#category": ("", "issuu", "user"), + "#class" : issuu.IssuuUserExtractor, + "#pattern" : issuu.IssuuPublicationExtractor.pattern, + "#count" : "> 25", +}, + +) diff --git a/test/results/itaku.py b/test/results/itaku.py new file mode 100644 index 00000000..ef3d3679 --- /dev/null +++ b/test/results/itaku.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import itaku + + +__tests__ = ( +{ + "#url" : "https://itaku.ee/profile/piku/gallery", + "#category": ("", "itaku", "gallery"), + "#class" : itaku.ItakuGalleryExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/[^/?#]+\.(jpg|png|gif)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://itaku.ee/images/100471", + "#category": ("", "itaku", "image"), + "#class" : itaku.ItakuImageExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/220504_oUNIAFT\.png", + "#count" : 1, + + "already_pinned" : None, + "blacklisted" : { + "blacklisted_tags": [], + "is_blacklisted" : False, + }, + "can_reshare" : True, + "date" : "dt:2022-05-05 19:21:17", + "date_added" : "2022-05-05T19:21:17.674148Z", + "date_edited" : "2022-05-25T14:37:46.220612Z", + "description" : "sketch from drawpile", + "extension" : "png", + "filename" : "220504_oUNIAFT", + "hotness_score" : float, + "id" : 100471, + "image" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT.png", + "image_xl" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT/lg.jpg", + "liked_by_you" : False, + "maturity_rating" : "SFW", + "num_comments" : int, + "num_likes" : int, + "num_reshares" : int, + "obj_tags" : 136446, + "owner" : 16775, + "owner_avatar" : "https://d1wmr8tlk3viaj.cloudfront.net/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_displayname": "Piku", + "owner_username" : "piku", + "reshared_by_you" : False, + "sections" : ["Fanart/Miku"], + "tags" : list, + "tags_character" : ["hatsune_miku"], + "tags_copyright" : ["vocaloid"], + "tags_general" : [ + "twintails", + "green_hair", + "flag", + "gloves", + "green_eyes", + "female", + "racing_miku", + ], + "title" : "Racing Miku 2022 Ver.", + "too_mature" : False, + "uncompressed_filesize": "0.62", + "video" : None, + "visibility" : "PUBLIC", +}, + +{ + "#url" : "https://itaku.ee/images/19465", + "#comment" : "video", + "#category": ("", "itaku", "image"), + "#class" : itaku.ItakuImageExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids/sleepy_af_OY5GHWw\.mp4", +}, + +) diff --git a/test/results/itchio.py b/test/results/itchio.py new file mode 100644 index 00000000..d2acfa61 --- /dev/null +++ b/test/results/itchio.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import itchio + + +__tests__ = ( +{ + "#url" : "https://sirtartarus.itch.io/a-craft-of-mine", + "#category": ("", "itchio", "game"), + "#class" : itchio.ItchioGameExtractor, + "#pattern" : r"https://\w+\.ssl\.hwcdn\.net/upload2/game/1983311/7723751\?", + "#count" : 1, + + "extension": "", + "filename" : "7723751", + "game" : { + "id" : 1983311, + "noun" : "game", + "title": "A Craft Of Mine", + "url" : "https://sirtartarus.itch.io/a-craft-of-mine", + }, + "user" : { + "id" : 4060052, + "name": "SirTartarus", + "url" : "https://sirtartarus.itch.io", + }, +}, + +) diff --git a/test/results/joyreactor.py b/test/results/joyreactor.py new file mode 100644 index 00000000..4c29c213 --- /dev/null +++ b/test/results/joyreactor.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://joyreactor.cc/tag/Advent+Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "http://joyreactor.com/tag/Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", +}, + +{ + "#url" : "http://joyreactor.com/tag/Dark+Souls+2/best", + "#comment" : "'best' rating (#3073)", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#count" : 4, +}, + +{ + "#url" : "http://joyreactor.cc/search/Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://joyreactor.com/search?q=Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://joyreactor.cc/user/hemantic", + "#category": ("reactor", "joyreactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://joyreactor.com/user/Tacoman123", + "#category": ("reactor", "joyreactor", "user"), + "#class" : reactor.ReactorUserExtractor, + "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", +}, + +{ + "#url" : "http://joyreactor.com/post/3721876", + "#comment" : "single image", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", + "#count" : 1, + "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", +}, + +{ + "#url" : "http://joyreactor.com/post/3713804", + "#comment" : "4 images", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", + "#count" : 4, + "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", +}, + +{ + "#url" : "http://joyreactor.com/post/3726210", + "#comment" : "gif / video", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", + "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", +}, + +{ + "#url" : "http://joyreactor.com/post/3668724", + "#comment" : "youtube embed", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "bf1666eddcff10c9b58f6be63fa94e4e13074214", + "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", +}, + +{ + "#url" : "http://joyreactor.cc/post/1299", + "#comment" : "'malformed' JSON", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", +}, + +) diff --git a/test/results/jpgfish.py b/test/results/jpgfish.py new file mode 100644 index 00000000..5aa4a126 --- /dev/null +++ b/test/results/jpgfish.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import jpgfish + + +__tests__ = ( +{ + "#url" : "https://jpg1.su/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, + "#pattern" : r"https://simp3\.jpg\.church/images/funnymeme\.jpg", + "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", + + "album" : "", + "extension": "jpg", + "filename" : "funnymeme", + "id" : "LecXGS", + "url" : "https://simp3.jpg.church/images/funnymeme.jpg", + "user" : "exearco", +}, + +{ + "#url" : "https://jpg.church/img/auCruA", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, + "#pattern" : r"https://simp2\.jpg\.church/hannahowo_00457\.jpg", + + "album": "401-500", +}, + +{ + "#url" : "https://jpeg.pet/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.pet/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.fishing/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.fish/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.church/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://jpg.fishing/a/gunggingnsk.N9OOI", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 114, +}, + +{ + "#url" : "https://jpg.fish/a/101-200.aNJ6A/", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 100, +}, + +{ + "#url" : "https://jpg.church/a/hannahowo.aNTdH/sub", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 606, +}, + +{ + "#url" : "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, +}, + +{ + "#url" : "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, +}, + +{ + "#url" : "https://jpg1.su/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://jpg.church/exearco/albums", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://jpeg.pet/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.pet/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.fishing/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.fish/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.church/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +) diff --git a/test/results/kabeuchi.py b/test/results/kabeuchi.py new file mode 100644 index 00000000..37b6a3d8 --- /dev/null +++ b/test/results/kabeuchi.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kabeuchi +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://kabe-uchiroom.com/mypage/?id=919865303848255493", + "#category": ("", "kabeuchi", "user"), + "#class" : kabeuchi.KabeuchiUserExtractor, + "#pattern" : r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", + "#count" : ">= 24", +}, + +{ + "#url" : "https://kabe-uchiroom.com/mypage/?id=123456789", + "#category": ("", "kabeuchi", "user"), + "#class" : kabeuchi.KabeuchiUserExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/keenspot.py b/test/results/keenspot.py new file mode 100644 index 00000000..9f756ad8 --- /dev/null +++ b/test/results/keenspot.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import keenspot + + +__tests__ = ( +{ + "#url" : "http://marksmen.keenspot.com/", + "#comment" : "link", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", +}, + +{ + "#url" : "http://barkercomic.keenspot.com/", + "#comment" : "id", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", +}, + +{ + "#url" : "http://crowscare.keenspot.com/", + "#comment" : "id v2", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", +}, + +{ + "#url" : "http://supernovas.keenspot.com/", + "#comment" : "ks", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", +}, + +{ + "#url" : "http://twokinds.keenspot.com/comic/1066/", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", +}, + +) diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py new file mode 100644 index 00000000..f5419f02 --- /dev/null +++ b/test/results/kemonoparty.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kemonoparty + + +__tests__ = ( +{ + "#url" : "https://kemono.party/fanbox/user/6993449", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyUserExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://kemono.party/patreon/user/881792?o=150", + "#comment" : "'max-posts' option, 'o' query parameter (#1674)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyUserExtractor, + "#options" : {"max-posts": 25}, + "#count" : "< 100", +}, + +{ + "#url" : "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyUserExtractor, +}, + +{ + "#url" : "https://kemono.party/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyUserExtractor, +}, + +{ + "#url" : "https://kemono.party/fanbox/user/6993449/post/506575", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono.party/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", + "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", + + "added" : "Wed, 06 May 2020 20:28:02 GMT", + "content" : str, + "count" : 1, + "date" : "dt:2019-08-11 02:09:04", + "edited" : None, + "embed" : dict, + "extension" : "jpeg", + "filename" : "P058kDFYus7DbqAkGlfWTlOr", + "hash" : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", + "id" : "506575", + "num" : 1, + "published" : "Sun, 11 Aug 2019 02:09:04 GMT", + "service" : "fanbox", + "shared_file": False, + "subcategory": "fanbox", + "title" : "c96取り置き", + "type" : "file", + "user" : "6993449", +}, + +{ + "#url" : "https://kemono.party/fanbox/user/7356311/post/802343", + "#comment" : "inline image (#1286)", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono\.party/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", + + "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", +}, + +{ + "#url" : "https://kemono.party/gumroad/user/trylsc/post/IURjT", + "#comment" : "kemono.party -> data.kemono.party", + "#category": ("", "kemonoparty", "gumroad"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono\.party/data/(a4/7b/a47bfe938d8c1682eef06e885927484cd8df1b.+\.jpg|c6/04/c6048f5067fd9dbfa7a8be565ac194efdfb6e4.+\.zip)", +}, + +{ + "#url" : "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", + "#comment" : "username (#1548, #1652)", + "#category": ("", "kemonoparty", "gumroad"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"metadata": True}, + + "username": "Kudalyn's Creations", +}, + +{ + "#url" : "https://kemono.party/patreon/user/4158582/post/32099982", + "#comment" : "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://kemono.party/patreon/user/4158582/post/32099982", + "#comment" : "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"duplicates": True}, + "#count" : 3, +}, + +{ + "#url" : "https://kemono.party/patreon/user/34134344/post/38129255", + "#comment" : "DMs (#2008)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"dms": True}, + + "dms": [{ + "body" : r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", + "date" : "2021-07-31 02:47:51.327865", +}], +}, + +{ + "#url" : "https://kemono.party/patreon/user/19623797/post/29035449", + "#comment" : "invalid file (#3510)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", + "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", +}, + +{ + "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://www.kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://beta.kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://kemono.party/discord/server/488668827274444803#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#count" : 4, + + "channel_name": "finish-work", +}, + +{ + "#url" : "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#pattern" : r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", + "#count" : ">= 2", + + "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", +}, + +{ + "#url" : "https://kemono.party/discord/server/315262215055736843/channel/315262215055736843#general", + "#comment" : "'inline' files", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#options" : {"image-filter": "type == 'inline'"}, + "#pattern" : r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", + "#range" : "1-5", + + "hash": "", +}, + +{ + "#url" : "https://kemono.party/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class" : kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count" : 13, +}, + +{ + "#url" : "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class" : kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count" : 13, +}, + +{ + "#url" : "https://kemono.party/favorites", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, + "#count" : 3, + "#sha1_url": "f4b5b796979bcba824af84206578c79101c7f0e1", +}, + +{ + "#url" : "https://kemono.party/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, + "#count" : 3, + "#sha1_url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f", +}, + +{ + "#url" : "https://kemono.su/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, + "#count" : 3, + "#sha1_url": "4be8e84cb384a907a8e7997baaf6287b451783b5", +}, + +) diff --git a/test/results/khinsider.py b/test/results/khinsider.py new file mode 100644 index 00000000..b680cab2 --- /dev/null +++ b/test/results/khinsider.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import khinsider + + +__tests__ = ( +{ + "#url" : "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", + "#category": ("", "khinsider", "soundtrack"), + "#class" : khinsider.KhinsiderSoundtrackExtractor, + "#pattern" : r"https?://vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", + "#count" : 1, + + "album" : { + "count" : 1, + "date" : "Sep 18th, 2016", + "name" : "Horizon Riders", + "platform": "Wii", + "size" : 26214400, + "type" : "Gamerip", + }, + "extension": "mp3", + "filename" : "Horizon Riders Wii - Full Soundtrack", +}, + +) diff --git a/test/results/kohlchan.py b/test/results/kohlchan.py new file mode 100644 index 00000000..479ed715 --- /dev/null +++ b/test/results/kohlchan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://kohlchan.net/a/res/4594.html", + "#category": ("lynxchan", "kohlchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count" : ">= 80", +}, + +{ + "#url" : "https://kohlchan.net/a/", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://kohlchan.net/a/2.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +{ + "#url" : "https://kohlchan.net/a/catalog.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/komikcast.py b/test/results/komikcast.py new file mode 100644 index 00000000..9a246009 --- /dev/null +++ b/test/results/komikcast.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import komikcast + + +__tests__ = ( +{ + "#url" : "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class" : komikcast.KomikcastChapterExtractor, + "#sha1_url" : "f6b43fbc027697749b3ea1c14931c83f878d7936", + "#sha1_metadata": "f3938e1aff9ad1f302f52447e9781b21f6da26d4", +}, + +{ + "#url" : "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class" : komikcast.KomikcastChapterExtractor, + "#sha1_url" : "efd00a9bd95461272d51990d7bc54b79ff3ff2e6", + "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", +}, + +{ + "#url" : "https://komikcast.site/komik/090-eko-to-issho/", + "#category": ("", "komikcast", "manga"), + "#class" : komikcast.KomikcastMangaExtractor, + "#sha1_url" : "19d3d50d532e84be6280a3d61ff0fd0ca04dd6b4", + "#sha1_metadata": "837a7e96867344ff59d840771c04c20dc46c0ab1", +}, + +{ + "#url" : "https://komikcast.me/tonari-no-kashiwagi-san/", + "#category": ("", "komikcast", "manga"), + "#class" : komikcast.KomikcastMangaExtractor, +}, + +) diff --git a/test/results/konachan.py b/test/results/konachan.py new file mode 100644 index 00000000..576acdaa --- /dev/null +++ b/test/results/konachan.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://konachan.com/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", + + "tags_artist" : "patata", + "tags_character": "clownpiece", + "tags_copyright": "touhou", + "tags_general" : str, +}, + +{ + "#url" : "https://konachan.net/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://konachan.com/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, + "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", +}, + +{ + "#url" : "https://konachan.com/post?tags=", + "#comment" : "empty 'tags' (#4354)", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://konachan.net/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://konachan.com/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, + "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", +}, + +{ + "#url" : "https://konachan.net/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://konachan.com/post/popular_by_month?month=11&year=2010", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, + "#count" : 20, +}, + +{ + "#url" : "https://konachan.com/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +{ + "#url" : "https://konachan.net/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/lensdump.py b/test/results/lensdump.py new file mode 100644 index 00000000..73f1fbdf --- /dev/null +++ b/test/results/lensdump.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lensdump + + +__tests__ = ( +{ + "#url" : "https://lensdump.com/a/1IhJr", + "#category": ("", "lensdump", "album"), + "#class" : lensdump.LensdumpAlbumExtractor, + "#pattern" : r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", + + "extension": "png", + "name" : str, + "num" : int, + "title" : str, + "url" : str, + "width" : int, +}, + +{ + "#url" : "https://lensdump.com/vstar925/albums", + "#category": ("", "lensdump", "albums"), + "#class" : lensdump.LensdumpAlbumsExtractor, +}, + +{ + "#url" : "https://lensdump.com/i/tyoAyM", + "#category": ("", "lensdump", "image"), + "#class" : lensdump.LensdumpImageExtractor, + "#pattern" : r"https://c\.l3n\.co/i/tyoAyM\.webp", + "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", + + "date" : "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename" : "tyoAyM", + "height" : 400, + "id" : "tyoAyM", + "title" : "MYOBI clovis bookcaseset", + "url" : "https://c.l3n.co/i/tyoAyM.webp", + "width" : 620, +}, + +) diff --git a/test/results/lesbianenergy.py b/test/results/lesbianenergy.py new file mode 100644 index 00000000..fffe8a0c --- /dev/null +++ b/test/results/lesbianenergy.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://lesbian.energy/@rerorero", + "#category": ("misskey", "lesbian.energy", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#pattern" : r"https://lesbian.energy/files/\w+", + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://lesbian.energy/@nano@mk.yopo.work", + "#category": ("misskey", "lesbian.energy", "user"), + "#class" : misskey.MisskeyUserExtractor, +}, + +{ + "#url" : "https://lesbian.energy/notes/995ig09wqy", + "#category": ("misskey", "lesbian.energy", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://lesbian.energy/notes/96ynd9w5kc", + "#category": ("misskey", "lesbian.energy", "note"), + "#class" : misskey.MisskeyNoteExtractor, +}, + +{ + "#url" : "https://lesbian.energy/my/favorites", + "#category": ("misskey", "lesbian.energy", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/lexica.py b/test/results/lexica.py new file mode 100644 index 00000000..07183f67 --- /dev/null +++ b/test/results/lexica.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lexica + + +__tests__ = ( +{ + "#url" : "https://lexica.art/?q=tree", + "#category": ("", "lexica", "search"), + "#class" : lexica.LexicaSearchExtractor, + "#pattern" : r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", + "#range" : "1-80", + "#count" : 80, + + "height" : int, + "id" : str, + "upscaled_height": int, + "upscaled_width" : int, + "userid" : str, + "width" : int, + "prompt" : { + "c" : int, + "grid" : bool, + "height" : int, + "id" : str, + "images" : list, + "initImage" : None, + "initImageStrength": None, + "model" : "lexica-aperture-v2", + "negativePrompt" : str, + "prompt" : str, + "seed" : str, + "timestamp" : r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", + "width" : int, + }, +}, + +) diff --git a/test/results/lightroom.py b/test/results/lightroom.py new file mode 100644 index 00000000..4c1a0532 --- /dev/null +++ b/test/results/lightroom.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lightroom + + +__tests__ = ( +{ + "#url" : "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", + "#category": ("", "lightroom", "gallery"), + "#class" : lightroom.LightroomGalleryExtractor, + "#count" : ">= 55", + + "title": "Sterne und Nachtphotos", + "user" : "Christian Schrang", +}, + +{ + "#url" : "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", + "#category": ("", "lightroom", "gallery"), + "#class" : lightroom.LightroomGalleryExtractor, + "#count" : ">= 180", + + "title": "HEBFC Snr/Res v Brighton", + "user" : "", +}, + +) diff --git a/test/results/livedoor.py b/test/results/livedoor.py new file mode 100644 index 00000000..7ab366f9 --- /dev/null +++ b/test/results/livedoor.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import livedoor + + +__tests__ = ( +{ + "#url" : "http://blog.livedoor.jp/zatsu_ke/", + "#category": ("", "livedoor", "blog"), + "#class" : livedoor.LivedoorBlogExtractor, + "#pattern" : r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", + "#range" : "1-50", + "#count" : 50, + "#archive" : False, + + "post" : { + "categories" : tuple, + "date" : "type:datetime", + "description": str, + "id" : int, + "tags" : list, + "title" : str, + "user" : "zatsu_ke", + }, + "filename": str, + "hash" : r"re:\w{4,}", + "num" : int, +}, + +{ + "#url" : "http://blog.livedoor.jp/uotapo/", + "#category": ("", "livedoor", "blog"), + "#class" : livedoor.LivedoorBlogExtractor, + "#range" : "1-5", + "#count" : 5, +}, + +{ + "#url" : "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", + "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", +}, + +{ + "#url" : "http://blog.livedoor.jp/amaumauma/archives/7835811.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "204bbd6a9db4969c50e0923855aeede04f2e4a62", + "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", +}, + +{ + "#url" : "http://blog.livedoor.jp/uotapo/archives/1050616939.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "4b5ab144b7309eb870d9c08f8853d1abee9946d2", + "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", +}, + +) diff --git a/test/results/lolibooru.py b/test/results/lolibooru.py new file mode 100644 index 00000000..f8750269 --- /dev/null +++ b/test/results/lolibooru.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://lolibooru.moe/post/show/281305/", + "#category": ("moebooru", "lolibooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"notes": True}, + "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", + + "notes": list, +}, + +{ + "#url" : "https://lolibooru.moe/post/show/287835", + "#category": ("moebooru", "lolibooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", + "#category": ("moebooru", "lolibooru", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/pool/show/239", + "#category": ("moebooru", "lolibooru", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/post/popular_recent", + "#category": ("moebooru", "lolibooru", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/loudbooru.py b/test/results/loudbooru.py new file mode 100644 index 00000000..58236b7d --- /dev/null +++ b/test/results/loudbooru.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://loudbooru.com/post/list/original_character/1", + "#category": ("shimmie2", "loudbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://loudbooru\.com/_images/[0-9a-f]{32}/\d+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://loudbooru.com/post/view/33828", + "#category": ("shimmie2", "loudbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://loudbooru\.com/_images/.+\.png", + "#sha1_content": "a4755f787ba23ae2aa297a46810f802ca9032739", + + "extension": "png", + "file_url" : "https://loudbooru.com/_images/ca2638d903c86e8337fe9aeb4974be88/33828%20-%202020%20artist%3Astikyfinkaz%20character%3Alisa_loud%20cover%20fanfiction%3Aplatz_eins%20frowning%20half-closed_eyes%20solo%20text%20title_card.png", + "filename" : "33828 - 2020 artist:stikyfinkaz character:lisa_loud cover fanfiction:platz_eins frowning half-closed_eyes solo text title_card", + "height" : 1920, + "id" : 33828, + "md5" : "ca2638d903c86e8337fe9aeb4974be88", + "tags" : "2020 artist:stikyfinkaz character:lisa_loud cover fanfiction:platz_eins frowning half-closed_eyes solo text title_card", + "width" : 1078, +}, + +) diff --git a/test/results/loungeunderwear.py b/test/results/loungeunderwear.py new file mode 100644 index 00000000..521de25c --- /dev/null +++ b/test/results/loungeunderwear.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://loungeunderwear.com/collections/apparel", + "#category": ("shopify", "loungeunderwear", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://de.loungeunderwear.com/products/ribbed-crop-top-black", + "#category": ("shopify", "loungeunderwear", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/luscious.py b/test/results/luscious.py new file mode 100644 index 00000000..d5429612 --- /dev/null +++ b/test/results/luscious.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import luscious +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://luscious.net/albums/okinami-no-koigokoro_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#pattern" : r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", + + "album" : { + "__typename" : "Album", + "audiences" : list, + "content" : "Hentai", + "cover" : r"re:https://\w+.luscious.net/.+/277031/", + "created" : 1479625853, + "created_by" : "NTRshouldbeillegal", + "date" : "dt:2016-11-20 07:10:53", + "description" : "Enjoy.", + "download_url" : r"re:/download/(r/)?824778/277031/", + "genres" : list, + "id" : 277031, + "is_manga" : True, + "labels" : list, + "language" : "English", + "like_status" : "none", + "modified" : int, + "permissions" : list, + "rating" : float, + "slug" : "okinami-no-koigokoro", + "status" : None, + "tags" : list, + "title" : "Okinami no Koigokoro", + "url" : "/albums/okinami-no-koigokoro_277031/", + "marked_for_deletion" : False, + "marked_for_processing" : False, + "number_of_animated_pictures": 0, + "number_of_favorites" : int, + "number_of_pictures" : 18, + }, + "aspect_ratio" : r"re:\d+:\d+", + "category" : "luscious", + "created" : int, + "date" : datetime.datetime, + "height" : int, + "id" : int, + "is_animated" : False, + "like_status" : "none", + "position" : int, + "resolution" : r"re:\d+x\d+", + "status" : None, + "tags" : list, + "thumbnail" : str, + "title" : str, + "width" : int, + "number_of_comments": int, + "number_of_favorites": int, +}, + +{ + "#url" : "https://luscious.net/albums/not-found_277035/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://members.luscious.net/albums/login-required_323871/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#count" : 64, +}, + +{ + "#url" : "https://www.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/list/", + "#category": ("", "luscious", "search"), + "#class" : luscious.LusciousSearchExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", + "#category": ("", "luscious", "search"), + "#class" : luscious.LusciousSearchExtractor, + "#pattern" : luscious.LusciousAlbumExtractor.pattern, + "#range" : "41-60", + "#count" : 20, +}, + +) diff --git a/test/results/mangadex.py b/test/results/mangadex.py new file mode 100644 index 00000000..c6a9e53a --- /dev/null +++ b/test/results/mangadex.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangadex +from gallery_dl import exception +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", +}, + +{ + "#url" : "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", + "#comment" : "oneshot", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#count" : 64, + "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", +}, + +{ + "#url" : "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", + "#comment" : "MANGA Plus (#1154)", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", + "#comment" : "'externalUrl', but still downloadable / 404 (#2503)", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment" : "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : ">= 5", + + "manga" : "Souten no Koumori", + "manga_id" : "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "title" : r"re:One[Ss]hot", + "volume" : 0, + "chapter" : 0, + "chapter_minor": "", + "chapter_id" : str, + "date" : datetime.datetime, + "lang" : str, + "language" : str, + "artist" : ["Arakawa Hiromu"], + "author" : ["Arakawa Hiromu"], + "status" : "completed", + "tags" : [ + "Oneshot", + "Historical", + "Action", + "Martial Arts", + "Drama", + "Tragedy", + ], +}, + +{ + "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment" : "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#options" : {"lang": "fr,it"}, + "#count" : 2, + + "manga" : "Souten no Koumori", + "lang" : r"re:fr|it", + "language": r"re:French|Italian", +}, + +{ + "#url" : "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#options" : {"lang": "en"}, + "#count" : ">= 100", +}, + +{ + "#url" : "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : ">= 20", +}, + +{ + "#url" : "https://mangadex.org/title/feed", + "#category": ("", "mangadex", "feed"), + "#class" : mangadex.MangadexFeedExtractor, +}, + +) diff --git a/test/results/mangafox.py b/test/results/mangafox.py new file mode 100644 index 00000000..dc0cc9a0 --- /dev/null +++ b/test/results/mangafox.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangafox +import datetime + + +__tests__ = ( +{ + "#url" : "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, + "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", + "#sha1_content" : "5c50c252dcf12ffecf68801f4db8a2167265f66c", +}, + +{ + "#url" : "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, +}, + +{ + "#url" : "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, +}, + +{ + "#url" : "https://fanfox.net/manga/kanojo_mo_kanojo", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, + "#pattern" : mangafox.MangafoxChapterExtractor.pattern, + "#count" : ">=60", + + "author" : "HIROYUKI", + "chapter" : int, + "chapter_minor" : r"re:^(\.\d+)?$", + "chapter_string": r"re:(v\d+/)?c\d+", + "date" : datetime.datetime, + "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", + "lang" : "en", + "language" : "English", + "manga" : "Kanojo mo Kanojo", + "tags" : [ + "Comedy", + "Romance", + "School Life", + "Shounen", + ], + "volume" : int, +}, + +{ + "#url" : "https://mangafox.me/manga/shangri_la_frontier", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, + "#pattern" : mangafox.MangafoxChapterExtractor.pattern, + "#count" : ">=45", +}, + +{ + "#url" : "https://m.fanfox.net/manga/sentai_daishikkaku", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, +}, + +) diff --git a/test/results/mangahere.py b/test/results/mangahere.py new file mode 100644 index 00000000..db17a6b0 --- /dev/null +++ b/test/results/mangahere.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangahere + + +__tests__ = ( +{ + "#url" : "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, + "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", + "#sha1_content" : "708d475f06893b88549cbd30df1e3f9428f2c884", +}, + +{ + "#url" : "https://www.mangahere.cc/manga/beastars/c196/1.html", + "#comment" : "URLs without HTTP scheme (#1070)", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, + "#pattern" : "https://zjcdn.mangahere.org/.*", +}, + +{ + "#url" : "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, +}, + +{ + "#url" : "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, +}, + +{ + "#url" : "https://www.mangahere.cc/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#count" : 71, + "#sha1_url" : "9c2e54ec42e9a87ad53096c328b33c90750af3e4", + "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", +}, + +{ + "#url" : "https://www.mangahere.cc/manga/hiyokoi/#50", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#sha1_url" : "654850570aa03825cd57e2ae2904af489602c523", + "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", +}, + +{ + "#url" : "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", + "#comment" : "adult filter (#556)", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#pattern" : mangahere.MangahereChapterExtractor.pattern, + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, +}, + +{ + "#url" : "https://m.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, +}, + +) diff --git a/test/results/mangakakalot.py b/test/results/mangakakalot.py new file mode 100644 index 00000000..b0b8bada --- /dev/null +++ b/test/results/mangakakalot.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangakakalot + + +__tests__ = ( +{ + "#url" : "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", + "#category": ("", "mangakakalot", "chapter"), + "#class" : mangakakalot.MangakakalotChapterExtractor, + "#pattern" : r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", + "#count" : 9, + "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", +}, + +{ + "#url" : "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", + "#category": ("", "mangakakalot", "chapter"), + "#class" : mangakakalot.MangakakalotChapterExtractor, +}, + +{ + "#url" : "https://ww3.mangakakalot.tv/manga/manga-jk986845", + "#category": ("", "mangakakalot", "manga"), + "#class" : mangakakalot.MangakakalotMangaExtractor, + "#pattern" : mangakakalot.MangakakalotChapterExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://mangakakalot.tv/manga/lk921810", + "#category": ("", "mangakakalot", "manga"), + "#class" : mangakakalot.MangakakalotMangaExtractor, +}, + +) diff --git a/test/results/mangalife.py b/test/results/mangalife.py new file mode 100644 index 00000000..80226afe --- /dev/null +++ b/test/results/mangalife.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangasee +import datetime + + +__tests__ = ( +{ + "#url" : "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", + "#category": ("", "mangalife", "chapter"), + "#class" : mangasee.MangaseeChapterExtractor, + "#pattern" : r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", + "#count" : 13, + + "author" : ["ODA Eiichiro"], + "chapter" : 1063, + "chapter_minor" : "", + "chapter_string": "110630", + "count" : 13, + "date" : "dt:2022-10-16 17:32:54", + "extension" : "png", + "filename" : r"re:1063-0\d\d", + "genre" : [ + "Action", + "Adventure", + "Comedy", + "Drama", + "Fantasy", + "Shounen", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "One Piece", + "page" : int, + "title" : "", +}, + +{ + "#url" : "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "#category": ("", "mangalife", "manga"), + "#class" : mangasee.MangaseeMangaExtractor, + "#pattern" : mangasee.MangaseeChapterExtractor.pattern, + "#count" : ">= 50", + + "author" : ["Canno"], + "chapter" : int, + "chapter_minor" : r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date" : datetime.datetime, + "genre" : [ + "Comedy", + "Romance", + "School Life", + "Seinen", + "Shoujo Ai", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "title" : "", +}, + +) diff --git a/test/results/manganelo.py b/test/results/manganelo.py new file mode 100644 index 00000000..b5772656 --- /dev/null +++ b/test/results/manganelo.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import manganelo + + +__tests__ = ( +{ + "#url" : "https://chapmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", + "#count" : 25, + "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", +}, + +{ + "#url" : "https://chapmanganelo.com/manga-ti107776/chapter-4", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", + "#count" : 45, + "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", +}, + +{ + "#url" : "https://chapmanganato.com/manga-no991297/chapter-8", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#count" : 20, + + "chapter" : 8, + "chapter_minor": "-1", +}, + +{ + "#url" : "https://readmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://manganelo.com/chapter/gamers/chapter_15", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://manganelo.com/chapter/gq921227/chapter_23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://chapmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, + "#pattern" : manganelo.ManganeloChapterExtractor.pattern, + "#count" : ">= 25", +}, + +{ + "#url" : "https://m.manganelo.com/manga-ti107776", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, + "#pattern" : manganelo.ManganeloChapterExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://readmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +{ + "#url" : "https://manganelo.com/manga/read_otome_no_teikoku", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +{ + "#url" : "https://manganelo.com/manga/ol921234/", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +) diff --git a/test/results/mangapark.py b/test/results/mangapark.py new file mode 100644 index 00000000..432f535f --- /dev/null +++ b/test/results/mangapark.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangapark +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, + "#pattern" : r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", + "#count" : 70, + + "artist" : [], + "author" : ["Amano Kozue"], + "chapter" : 60, + "chapter_id" : 6710214, + "chapter_minor": ".2", + "count" : 70, + "date" : "dt:2022-01-15 09:25:03", + "extension" : "jpeg", + "filename" : str, + "genre" : [ + "adventure", + "comedy", + "drama", + "sci_fi", + "shounen", + "slice_of_life", + ], + "lang" : "en", + "language" : "English", + "manga" : "Aria", + "manga_id" : 114972, + "page" : int, + "source" : "Koala", + "title" : "Special Navigation - Aquaria Ii", + "volume" : 12, +}, + +{ + "#url" : "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.net/title/114972-aria", + "#comment" : "'source' option", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, + "#pattern" : mangapark.MangaparkChapterExtractor.pattern, + "#count" : 141, + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": str, + "date" : datetime.datetime, + "lang" : "en", + "language" : "English", + "manga_id" : 114972, + "source" : r"re:Horse|Koala", + "source_id" : int, + "title" : str, + "volume" : int, +}, + +{ + "#url" : "https://mangapark.net/title/114972-aria", + "#comment" : "'source' option", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, + "#options" : {"source": "koala"}, + "#pattern" : mangapark.MangaparkChapterExtractor.pattern, + "#count" : 70, + + "source" : "Koala", + "source_id": 15150116, +}, + +{ + "#url" : "https://mangapark.com/title/114972-", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.com/title/114972", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.com/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.org/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.io/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.me/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +) diff --git a/test/results/mangaread.py b/test/results/mangaread.py new file mode 100644 index 00000000..40fe4e21 --- /dev/null +++ b/test/results/mangaread.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangaread +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 11, + + "manga" : "One Piece", + "title" : "", + "chapter" : 1053, + "chapter_minor": ".3", + "tags" : ["Oda Eiichiro"], + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 9, + + "manga" : "Kanan-sama wa Akumade Choroi", + "title" : "", + "chapter" : 10, + "chapter_minor": "", + "tags" : list, + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", + "#comment" : "^^ no whitespace", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 6, + + "manga" : "Above All Gods", + "title" : "", + "chapter" : 146, + "chapter_minor": ".5", + "tags" : list, + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+(-.+)?/", + "#count" : ">= 13", + + "manga" : "Kanan-sama wa Akumade Choroi", + "author" : ["nonco"], + "artist" : ["nonco"], + "type" : "Manga", + "genres" : [ + "Comedy", + "Romance", + "Shounen", + "Supernatural", + ], + "rating" : float, + "release" : 2022, + "status" : "OnGoing", + "lang" : "en", + "language" : "English", + "manga_alt" : list, + "description": str, +}, + +{ + "#url" : "https://www.mangaread.org/manga/one-piece", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", + "#count" : ">= 1066", + + "manga" : "One Piece", + "author" : ["Oda Eiichiro"], + "artist" : ["Oda Eiichiro"], + "type" : "Manga", + "genres" : list, + "rating" : float, + "release" : 1997, + "status" : "OnGoing", + "lang" : "en", + "language" : "English", + "manga_alt" : ["One Piece"], + "description": str, +}, + +{ + "#url" : "https://www.mangaread.org/manga/doesnotexist", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#exception": exception.HttpError, +}, + +) diff --git a/test/results/mangasee.py b/test/results/mangasee.py new file mode 100644 index 00000000..8cf51d4e --- /dev/null +++ b/test/results/mangasee.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangasee +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", + "#category": ("", "mangasee", "chapter"), + "#class" : mangasee.MangaseeChapterExtractor, + "#pattern" : r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", + "#count" : 8, + + "author" : ["NARUMI Naru"], + "chapter" : 4, + "chapter_minor" : ".5", + "chapter_string": "100045", + "count" : 8, + "date" : "dt:2020-01-20 21:52:53", + "extension" : "png", + "filename" : r"re:0004\.5-00\d", + "genre" : [ + "Comedy", + "Fantasy", + "Harem", + "Romance", + "Shounen", + "Supernatural", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Tokyo Innocent", + "page" : int, + "title" : "", +}, + +{ + "#url" : "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "#category": ("", "mangasee", "manga"), + "#class" : mangasee.MangaseeMangaExtractor, + "#pattern" : mangasee.MangaseeChapterExtractor.pattern, + "#count" : ">= 17", + + "author" : ["TAKASE Masaya"], + "chapter" : int, + "chapter_minor" : r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date" : datetime.datetime, + "genre" : [ + "Comedy", + "Romance", + "School Life", + "Shounen", + "Slice of Life", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "title" : "", +}, + +) diff --git a/test/results/mangoxo.py b/test/results/mangoxo.py new file mode 100644 index 00000000..b7c392a4 --- /dev/null +++ b/test/results/mangoxo.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangoxo + + +__tests__ = ( +{ + "#url" : "https://www.mangoxo.com/album/lzVOv1Q9", + "#category": ("", "mangoxo", "album"), + "#class" : mangoxo.MangoxoAlbumExtractor, + "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", + + "channel": { + "id" : "gaxO16d8", + "name" : "Phoenix", + "cover": str, + }, + "album" : { + "id" : "lzVOv1Q9", + "name" : r"re:池永康晟 Ikenaga Yasunari 透出古朴", + "date" : "dt:2019-03-22 14:42:00", + "description": str, + }, + "id" : int, + "num" : int, + "count" : 65, +}, + +{ + "#url" : "https://www.mangoxo.com/phoenix/album", + "#category": ("", "mangoxo", "channel"), + "#class" : mangoxo.MangoxoChannelExtractor, + "#pattern" : mangoxo.MangoxoAlbumExtractor.pattern, + "#range" : "1-30", + "#count" : "> 20", +}, + +) diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py new file mode 100644 index 00000000..a7001fd0 --- /dev/null +++ b/test/results/mastodonsocial.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://mastodon.social/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#pattern" : r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range" : "1-60", + "#count" : 60, +}, + +{ + "#url" : "https://mastodon.social/@yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#pattern" : r"https://mastodon\.social/media_proxy/\d+/original", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://mastodon.social/@id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/web/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/bookmarks", + "#category": ("mastodon", "mastodon.social", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, + "#count" : ">= 20", + "#extractor": False, +}, + +{ + "#url" : "https://mastodon.social/users/id:10843/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://mastodon.social/@jk/103794036899778366", + "#category": ("mastodon", "mastodon.social", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#count" : 4, + + "count": 4, + "num" : int, +}, + +) diff --git a/test/results/mememuseum.py b/test/results/mememuseum.py new file mode 100644 index 00000000..6a93d3f2 --- /dev/null +++ b/test/results/mememuseum.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://meme.museum/post/list/animated/1", + "#category": ("shimmie2", "mememuseum", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://meme\.museum/_images/\w+/\d+%20-%20", + "#count" : ">= 30", +}, + +{ + "#url" : "https://meme.museum/post/view/10243", + "#category": ("shimmie2", "mememuseum", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://meme\.museum/_images/105febebcd5ca791ee332adc49971f78/10243%20-%20g%20beard%20open_source%20richard_stallman%20stallman%20tagme%20text\.jpg", + "#sha1_content": "45565f3f141fc960a8ae1168b80e718a494c52d2", + + "extension" : "jpg", + "file_url" : "https://meme.museum/_images/105febebcd5ca791ee332adc49971f78/10243%20-%20g%20beard%20open_source%20richard_stallman%20stallman%20tagme%20text.jpg", + "filename" : "10243 - g beard open_source richard_stallman stallman tagme text", + "height" : 451, + "id" : 10243, + "md5" : "105febebcd5ca791ee332adc49971f78", + "size" : 0, + "subcategory": "post", + "tags" : "/g/ beard open_source richard_stallman stallman tagme text", + "width" : 480, +}, + +) diff --git a/test/results/michaelscameras.py b/test/results/michaelscameras.py new file mode 100644 index 00000000..42c76bbc --- /dev/null +++ b/test/results/michaelscameras.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://michaels.com.au/collections/microphones", + "#category": ("shopify", "michaelscameras", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", + "#category": ("shopify", "michaelscameras", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/misskeyio.py b/test/results/misskeyio.py new file mode 100644 index 00000000..3a28ae87 --- /dev/null +++ b/test/results/misskeyio.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://misskey.io/@lithla", + "#category": ("misskey", "misskey.io", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#pattern" : r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://misskey.io/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.io", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://misskey.io/@blooddj@pawoo.net/following", + "#category": ("misskey", "misskey.io", "following"), + "#class" : misskey.MisskeyFollowingExtractor, + "#count" : ">= 6", + "#extractor": False, +}, + +{ + "#url" : "https://misskey.io/notes/9bhqfo835v", + "#category": ("misskey", "misskey.io", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#pattern" : r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", + "#count" : 4, +}, + +{ + "#url" : "https://misskey.io/notes/9brq7z1re6", + "#category": ("misskey", "misskey.io", "note"), + "#class" : misskey.MisskeyNoteExtractor, +}, + +{ + "#url" : "https://misskey.io/my/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +{ + "#url" : "https://misskey.io/api/i/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/modcloth.py b/test/results/modcloth.py new file mode 100644 index 00000000..67ad082e --- /dev/null +++ b/test/results/modcloth.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://modcloth.com/collections/shoes", + "#category": ("shopify", "modcloth", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://modcloth.com/collections/shoes/products/heidii-brn", + "#category": ("shopify", "modcloth", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/myhentaigallery.py b/test/results/myhentaigallery.py new file mode 100644 index 00000000..6ddc2c2f --- /dev/null +++ b/test/results/myhentaigallery.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import myhentaigallery + + +__tests__ = ( +{ + "#url" : "https://myhentaigallery.com/gallery/thumbnails/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, + "#pattern" : r"https://images.myhentaicomics\.com/imagesgallery/images/[^/]+/original/\d+\.jpg", + + "artist" : list, + "count" : 11, + "gallery_id": 16247, + "group" : list, + "parodies" : list, + "tags" : ["Giantess"], + "title" : "Attack Of The 50ft Woman 1", +}, + +{ + "#url" : "https://myhentaigallery.com/gallery/show/16247/1", + "#category": ("", "myhentaigallery", "gallery"), + "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, +}, + +) diff --git a/test/results/myportfolio.py b/test/results/myportfolio.py new file mode 100644 index 00000000..71851293 --- /dev/null +++ b/test/results/myportfolio.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import myportfolio +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#sha1_url" : "acea0690c76db0e5cf267648cefd86e921bc3499", + "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", +}, + +{ + "#url" : "https://andrewling.myportfolio.com/", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#pattern" : r"https://andrewling\.myportfolio\.com/[^/?#+]+$", + "#count" : ">= 6", +}, + +{ + "#url" : "https://stevenilousphotography.myportfolio.com/society", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", + "#comment" : "custom domain", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#count" : 3, +}, + +{ + "#url" : "myportfolio:https://tooco.com.ar/", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#pattern" : myportfolio.MyportfolioGalleryExtractor.pattern, + "#count" : ">= 40", +}, + +) diff --git a/test/results/naver.py b/test/results/naver.py new file mode 100644 index 00000000..81d18efd --- /dev/null +++ b/test/results/naver.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import naver + + +__tests__ = ( +{ + "#url" : "https://blog.naver.com/rlfqjxm0/221430673006", + "#category": ("", "naver", "post"), + "#class" : naver.NaverPostExtractor, + "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", +}, + +{ + "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class" : naver.NaverPostExtractor, + "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", +}, + +{ + "#url" : "https://blog.naver.com/gukjung", + "#category": ("", "naver", "blog"), + "#class" : naver.NaverBlogExtractor, + "#pattern" : naver.NaverPostExtractor.pattern, + "#range" : "1-12", + "#count" : 12, +}, + +{ + "#url" : "https://blog.naver.com/PostList.nhn?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class" : naver.NaverBlogExtractor, + "#pattern" : naver.NaverPostExtractor.pattern, + "#range" : "1-12", + "#count" : 12, +}, + +) diff --git a/test/results/naverwebtoon.py b/test/results/naverwebtoon.py new file mode 100644 index 00000000..889e5bce --- /dev/null +++ b/test/results/naverwebtoon.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import naverwebtoon + + +__tests__ = ( +{ + "#url" : "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count" : 14, + "#sha1_url" : "47a956ba8c7a837213d5985f50c569fcff986f75", + "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", + + "author" : ["김규삼"], + "artist" : ["김규삼"], + "comic" : "N의등대-눈의등대", + "count" : 14, + "episode" : "1", + "extension": "jpg", + "num" : int, + "tags" : [ + "스릴러", + "완결무료", + "완결스릴러", + ], + "title" : "n의 등대 - 눈의 등대 1화", + "title_id" : "26458", +}, + +{ + "#url" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", + "#count" : 1, + + "author" : ["kemi****"], + "artist" : [], + "comic" : "우니 모두의 이야기", + "count" : 1, + "episode" : "1", + "extension": "jpeg", + "filename" : "upload_7149856273586337846", + "num" : 1, + "tags" : [ + "일상툰", + "우니모두의이야기", + "퇴사", + "입사", + "신입사원", + "사회초년생", + "회사원", + "20대", + ], + "title" : "퇴사하다", + "title_id" : "765124", +}, + +{ + "#url" : "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", + "#count" : 1, +}, + +{ + "#url" : "https://comic.naver.com/webtoon/list?titleId=22073", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : 32, +}, + +{ + "#url" : "https://comic.naver.com/challenge/list?titleId=765124", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : 25, +}, + +{ + "#url" : "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : ">= 12", +}, + +) diff --git a/test/results/newgrounds.py b/test/results/newgrounds.py new file mode 100644 index 00000000..3dac724d --- /dev/null +++ b/test/results/newgrounds.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import newgrounds + + +__tests__ = ( +{ + "#url" : "https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#sha1_url" : "57f182bcbbf2612690c3a54f16ffa1da5105245e", + "#sha1_content": "8f395e08333eb2457ba8d8b715238f8910221365", + + "artist" : ["tomfulp"], + "comment" : r"re:Consider this the bottom threshold for ", + "date" : "dt:2009-06-04 14:44:05", + "description": r"re:Consider this the bottom threshold for ", + "favorites" : int, + "filename" : "94_tomfulp_ryu-is-hawt", + "height" : 476, + "index" : 94, + "rating" : "e", + "score" : float, + "tags" : [ + "ryu", + "streetfighter", + ], + "title" : "Ryu is Hawt", + "type" : "article", + "user" : "tomfulp", + "width" : 447, +}, + +{ + "#url" : "https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#sha1_url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", +}, + +{ + "#url" : "https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#count" : 2, + "#sha1_url": "84eec95e663041a80630df72719f231e157e5f5d", +}, + +{ + "#url" : "https://www.newgrounds.com/art/view/kekiiro/red", + "#comment" : "'adult' rated (#2456)", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#options" : {"username": None}, + "#count" : 1, +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/595355", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/alternate/564000/564957_alternate_31\.mp4\?1359712249", + + "artist" : [ + "kickinthehead", + "danpaladin", + "tomfulp", + ], + "comment" : r"re:My fan trailer for Alien Hominid HD!", + "date" : "dt:2013-02-01 09:50:49", + "description": "Fan trailer for Alien Hominid HD!", + "favorites" : int, + "filename" : "564957_alternate_31", + "index" : 595355, + "rating" : "e", + "score" : float, + "tags" : [ + "alienhominid", + "trailer", + ], + "title" : "Alien Hominid Fan Trailer", + "type" : "movie", + "user" : "kickinthehead", +}, + +{ + "#url" : "https://www.newgrounds.com/audio/listen/609768", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#sha1_url": "f4c5490ae559a3b05e46821bb7ee834f93a43c95", + + "artist" : [ + "zj", + "tomfulp", + ], + "comment" : r"""re:RECORDED 12-09-2014 + +From The ZJ "Late """, + "date" : "dt:2015-02-23 19:31:59", + "description": "From The ZJ Report Show!", + "favorites" : int, + "index" : 609768, + "rating" : "", + "score" : float, + "tags" : [ + "fulp", + "interview", + "tom", + "zj", + ], + "title" : "ZJ Interviews Tom Fulp!", + "type" : "music.song", + "user" : "zj", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/161181/format/flash", + "#comment" : "flash animation (#1257)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/161000/161181_ddautta_mask__550x281_\.swf\?f1081628129", + + "type": "movie", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/758545", + "#comment" : "format selection (#1729)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"format": "720p"}, + "#pattern" : r"https://uploads\.ungrounded\.net/alternate/1482000/1482860_alternate_102516\.720p\.mp4\?\d+", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/717744", + "#comment" : "'adult' rated (#2456)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"username": None}, + "#count" : 1, +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/829032", + "#comment" : "flash game", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/829000/829032_picovsbeardx\.swf\?f1641968445", + "#range" : "1", + + "artist" : [ + "dungeonation", + "carpetbakery", + "animalspeakandrews", + "bill", + "chipollo", + "dylz49", + "gappyshamp", + "pinktophat", + "rad", + "shapeshiftingblob", + "tomfulp", + "voicesbycorey", + "psychogoldfish", + ], + "comment" : r"re:The children are expendable. Take out the ", + "date" : "dt:2022-01-10 23:00:57", + "description": "Bloodshed in The Big House that Blew...again!", + "favorites" : int, + "index" : 829032, + "post_url" : "https://www.newgrounds.com/portal/view/829032", + "rating" : "m", + "score" : float, + "tags" : [ + "assassin", + "boyfriend", + "darnell", + "nene", + "pico", + "picos-school", + ], + "title" : "PICO VS BEAR DX", + "type" : "game", + "url" : "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf?f1641968445", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/art", + "#category": ("", "newgrounds", "art"), + "#class" : newgrounds.NewgroundsArtExtractor, + "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, + "#count" : ">= 3", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/audio", + "#category": ("", "newgrounds", "audio"), + "#class" : newgrounds.NewgroundsAudioExtractor, + "#pattern" : r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3", + "#count" : ">= 4", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/movies", + "#category": ("", "newgrounds", "movies"), + "#class" : newgrounds.NewgroundsMoviesExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/games", + "#category": ("", "newgrounds", "games"), + "#class" : newgrounds.NewgroundsGamesExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com", + "#category": ("", "newgrounds", "user"), + "#class" : newgrounds.NewgroundsUserExtractor, + "#pattern" : "https://tomfulp.newgrounds.com/art$", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com", + "#category": ("", "newgrounds", "user"), + "#class" : newgrounds.NewgroundsUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "https://tomfulp.newgrounds.com/(art|audio|movies)$", + "#count" : 3, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/art", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/audio", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/movies", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/following", + "#category": ("", "newgrounds", "following"), + "#class" : newgrounds.NewgroundsFollowingExtractor, + "#pattern" : newgrounds.NewgroundsUserExtractor.pattern, + "#range" : "76-125", + "#count" : 50, +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/art?terms=tree", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, + "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, + "#range" : "1-10", + "#count" : 10, + + "search_tags": "tree", +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/movies?terms=tree", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, +}, + +) diff --git a/test/results/nhentai.py b/test/results/nhentai.py new file mode 100644 index 00000000..946acb3a --- /dev/null +++ b/test/results/nhentai.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nhentai + + +__tests__ = ( +{ + "#url" : "https://nhentai.net/g/147850/", + "#category": ("", "nhentai", "gallery"), + "#class" : nhentai.NhentaiGalleryExtractor, + "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", + + "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", + "title_en" : str, + "title_ja" : str, + "gallery_id": 147850, + "media_id" : 867789, + "count" : 16, + "date" : 1446050915, + "scanlator" : "", + "artist" : ["morris"], + "group" : list, + "parody" : list, + "characters": list, + "tags" : list, + "type" : "manga", + "lang" : "en", + "language" : "English", + "width" : int, + "height" : int, +}, + +{ + "#url" : "https://nhentai.net/tag/sole-female/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, + "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://nhentai.net/artist/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/group/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/parody/touhou-project/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/character/patchouli-knowledge/popular", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/category/doujinshi/popular-today", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/language/english/popular-week", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/search/?q=touhou", + "#category": ("", "nhentai", "search"), + "#class" : nhentai.NhentaiSearchExtractor, + "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://nhentai.net/favorites/", + "#category": ("", "nhentai", "favorite"), + "#class" : nhentai.NhentaiFavoriteExtractor, +}, + +) diff --git a/test/results/nijie.py b/test/results/nijie.py new file mode 100644 index 00000000..b2b10a70 --- /dev/null +++ b/test/results/nijie.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nijie +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://nijie.info/members.php?id=44", + "#category": ("Nijie", "nijie", "user"), + "#class" : nijie.NijieUserExtractor, +}, + +{ + "#url" : "https://nijie.info/members_illust.php?id=44", + "#category": ("Nijie", "nijie", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#sha1_url": "1553e5144df50a676f5947d02469299b401ad6c0", + + "artist_id" : 44, + "artist_name": "ED", + "date" : datetime.datetime, + "description": str, + "extension" : "jpg", + "filename" : str, + "image_id" : int, + "num" : int, + "tags" : list, + "title" : str, + "url" : r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", + "user_id" : 44, + "user_name" : "ED", +}, + +{ + "#url" : "https://nijie.info/members_illust.php?id=43", + "#category": ("Nijie", "nijie", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://nijie.info/members_dojin.php?id=6782", + "#category": ("Nijie", "nijie", "doujin"), + "#class" : nijie.NijieDoujinExtractor, + "#count" : ">= 18", + + "user_id" : 6782, + "user_name": "ジョニー＠アビオン村", +}, + +{ + "#url" : "https://nijie.info/user_like_illust_view.php?id=44", + "#category": ("Nijie", "nijie", "favorite"), + "#class" : nijie.NijieFavoriteExtractor, + "#count" : ">= 16", + + "user_id" : 44, + "user_name": "ED", +}, + +{ + "#url" : "https://nijie.info/history_nuita.php?id=728995", + "#category": ("Nijie", "nijie", "nuita"), + "#class" : nijie.NijieNuitaExtractor, + "#range" : "1-10", + "#count" : 10, + + "user_id" : 728995, + "user_name": "莚", +}, + +{ + "#url" : "https://nijie.info/like_user_view.php", + "#category": ("Nijie", "nijie", "feed"), + "#class" : nijie.NijieFeedExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://nijie.info/like_my.php", + "#category": ("Nijie", "nijie", "followed"), + "#class" : nijie.NijiefollowedExtractor, +}, + +{ + "#url" : "https://nijie.info/view.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, + "#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743", + "#sha1_metadata": "41da5d0e178b04f01fe72460185df52fadc3c91b", + "#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", +}, + +{ + "#url" : "https://nijie.info/view.php?id=70724", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://nijie.info/view_popup.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, +}, + +) diff --git a/test/results/nitter1d4us.py b/test/results/nitter1d4us.py new file mode 100644 index 00000000..4c6c3d12 --- /dev/null +++ b/test/results/nitter1d4us.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.1d4.us/supernaturepics", + "#category": ("nitter", "nitter.1d4.us", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#range" : "1", + + "user": {"id": "2976459548"}, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/with_replies", + "#category": ("nitter", "nitter.1d4.us", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/media", + "#category": ("nitter", "nitter.1d4.us", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/search", + "#category": ("nitter", "nitter.1d4.us", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/playpokemon/status/1263832915173048321", + "#comment" : "content with emoji, newlines, hashtags (#338)", + "#category": ("nitter", "nitter.1d4.us", "tweet"), + "#class" : nitter.NitterTweetExtractor, + + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! + +You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! + +❓🎁➡️ """, +}, + +{ + "#url" : "https://nitter.1d4.us/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("nitter", "nitter.1d4.us", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"https://nitter\.1d4\.us/pic/orig/enc/bWVkaWEvRWFL\w+LmpwZw==", + "#count" : 4, + + "filename": r"re:EaK.{12}", +}, + +) diff --git a/test/results/nitterit.py b/test/results/nitterit.py new file mode 100644 index 00000000..79d6937f --- /dev/null +++ b/test/results/nitterit.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.it/POTUS/status/1639409307878928384", + "#category": ("nitter", "nitter.it", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 0, +}, + +) diff --git a/test/results/nitterkavinrocks.py b/test/results/nitterkavinrocks.py new file mode 100644 index 00000000..22a72e2e --- /dev/null +++ b/test/results/nitterkavinrocks.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548", + "#category": ("nitter", "nitter.kavin.rocks", "tweets"), + "#class" : nitter.NitterTweetsExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/with_replies", + "#category": ("nitter", "nitter.kavin.rocks", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/media", + "#category": ("nitter", "nitter.kavin.rocks", "media"), + "#class" : nitter.NitterMediaExtractor, + "#pattern" : r"https://nitter\.kavin\.rocks/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/search", + "#category": ("nitter", "nitter.kavin.rocks", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/ed1conf/status/1163841619336007680", + "#comment" : "Nitter tweet (#890)", + "#category": ("nitter", "nitter.kavin.rocks", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url" : "e115bd1c86c660064e392b05269bbcafcd8c8b7a", + "#sha1_content": "f29501e44d88437fe460f5c927b7543fda0f6e34", +}, + +) diff --git a/test/results/nitterlacontrevoiefr.py b/test/results/nitterlacontrevoiefr.py new file mode 100644 index 00000000..b2a81ec1 --- /dev/null +++ b/test/results/nitterlacontrevoiefr.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#pattern" : r"https://nitter\.lacontrevoie\.fr/pic/orig/media%2FCGMNYZvW0AIVoom\.jpg", + "#range" : "1", + "#sha1_url": "54f4b55f2099dcc248f3fb7bfacf1349e08d8e2d", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/with_replies", + "#category": ("nitter", "nitter.lacontrevoie.fr", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/media", + "#category": ("nitter", "nitter.lacontrevoie.fr", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/search", + "#category": ("nitter", "nitter.lacontrevoie.fr", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/894001459754180609", + "#comment" : "4 images", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url": "9c51b3a4a1114535eb9b168bba97ad95db0d59ff", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1065692031626829824", + "#comment" : "video", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"ytdl:https://nitter\.lacontrevoie\.fr/video/[0-9A-F]{10,}/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1065691868439007232%2Fpu%2Fpl%2Fnv8hUQC1R0SjhzcZ.m3u8%3Ftag%3D5", + + "extension": "mp4", + "filename" : "nv8hUQC1R0SjhzcZ", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1460044411165888515", + "#comment" : "deleted quote tweet (#2225)", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1486373748911575046", + "#comment" : "'Misleading' content", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 4, +}, + +) diff --git a/test/results/nitternet.py b/test/results/nitternet.py new file mode 100644 index 00000000..2a1cccf6 --- /dev/null +++ b/test/results/nitternet.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter +import datetime + + +__tests__ = ( +{ + "#url" : "https://nitter.net/supernaturepics", + "#category": ("nitter", "nitter.net", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", + "#count" : 20, + + "author" : { + "name": "supernaturepics", + "nick": "Nature Pictures", + }, + "comments": int, + "content" : str, + "count" : 1, + "date" : datetime.datetime, + "likes" : int, + "quotes" : int, + "retweets": int, + "tweet_id": r"re:\d+", + "user" : { + "date" : "dt:2015-01-12 10:25:00", + "description" : "The very best nature pictures.", + "favourites_count": int, + "followers_count" : int, + "friends_count" : int, + "id" : "2976459548", + "name" : "supernaturepics", + "nick" : "Nature Pictures", + "profile_banner" : "https://nitter.net/pic/https%3A%2F%2Fpbs.twimg.com%2Fprofile_banners%2F2976459548%2F1421058583%2F1500x500", + "profile_image" : "https://nitter.net/pic/pbs.twimg.com%2Fprofile_images%2F554585280938659841%2FFLVAlX18.jpeg", + "statuses_count" : 1568, + "verified" : False, + }, +}, + +{ + "#url" : "https://nitter.net/supernaturepics/with_replies", + "#category": ("nitter", "nitter.net", "replies"), + "#class" : nitter.NitterRepliesExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/media", + "#category": ("nitter", "nitter.net", "media"), + "#class" : nitter.NitterMediaExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/search", + "#category": ("nitter", "nitter.net", "search"), + "#class" : nitter.NitterSearchExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/status/604341487988576256", + "#category": ("nitter", "nitter.net", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url" : "3f2b64e175bf284aa672c3bb53ed275e470b919a", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", + + "comments" : 19, + "content" : "Big Wedeene River, Canada", + "count" : 1, + "date" : "dt:2015-05-29 17:40:00", + "extension": "jpg", + "filename" : "CGMNYZvW0AIVoom", + "likes" : int, + "num" : 1, + "quotes" : 10, + "retweets" : int, + "tweet_id" : "604341487988576256", + "url" : "https://nitter.net/pic/orig/media%2FCGMNYZvW0AIVoom.jpg", + "user" : { + "name": "supernaturepics", + "nick": "Nature Pictures", + }, +}, + +{ + "#url" : "https://nitter.net/StobiesGalaxy/status/1270755918330896395", + "#comment" : "'quoted' option (#854)", + "#category": ("nitter", "nitter.net", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#options" : {"quoted": True}, + "#pattern" : r"https://nitter\.net/pic/orig/media%2FEa[KG].+\.jpg", + "#count" : 8, +}, + +) diff --git a/test/results/nitterunixfoxeu.py b/test/results/nitterunixfoxeu.py new file mode 100644 index 00000000..d9b216e2 --- /dev/null +++ b/test/results/nitterunixfoxeu.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.unixfox.eu/supernaturepics", + "#category": ("nitter", "nitter.unixfox.eu", "tweets"), + "#class" : nitter.NitterTweetsExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/with_replies", + "#category": ("nitter", "nitter.unixfox.eu", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/media", + "#category": ("nitter", "nitter.unixfox.eu", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/search", + "#category": ("nitter", "nitter.unixfox.eu", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/web/status/1170041925560258560", + "#comment" : "Reply to deleted tweet (#403, #838)", + "#category": ("nitter", "nitter.unixfox.eu", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"https://nitter\.unixfox\.eu/pic/orig/media%2FEDzS7VrU0AAFL4_\.jpg", +}, + +{ + "#url" : "https://nitter.unixfox.eu/mightbecurse/status/1492954264909479936", + "#comment" : "age-restricted (#2354)", + "#category": ("nitter", "nitter.unixfox.eu", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 1, + + "date": "dt:2022-02-13 20:10:00", +}, + +) diff --git a/test/results/nozomi.py b/test/results/nozomi.py new file mode 100644 index 00000000..042c7920 --- /dev/null +++ b/test/results/nozomi.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nozomi + + +__tests__ = ( +{ + "#url" : "https://nozomi.la/post/3649262.html", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", + "#sha1_url" : "e5525e717aec712843be8b88592d6406ae9e60ba", + "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", + + "artist" : ["hammer (sunset beach)"], + "character": ["patchouli knowledge"], + "copyright": ["touhou"], + "dataid" : r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", + "date" : "dt:2016-07-26 02:32:03", + "extension": "webp", + "filename" : str, + "height" : 768, + "is_video" : False, + "postid" : 3649262, + "tags" : list, + "type" : "jpg", + "url" : str, + "width" : 1024, +}, + +{ + "#url" : "https://nozomi.la/post/25588032.html", + "#comment" : "multiple images per post", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#count" : 7, + "#sha1_url" : "fb956ccedcf2cf509739d26e2609e910244aa56c", + "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", +}, + +{ + "#url" : "https://nozomi.la/post/130309.html", + "#comment" : "empty 'date' (#1163)", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + + "date": None, +}, + +{ + "#url" : "https://nozomi.la/post/1647.html", + "#comment" : "gif", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", + "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", +}, + +{ + "#url" : "https://nozomi.la/post/2269847.html", + "#comment" : "video", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", + "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", +}, + +{ + "#url" : "https://nozomi.la/", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/index-2.html", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/index-Popular-33.html", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/tag/3:1_aspect_ratio-1.html", + "#category": ("", "nozomi", "tag"), + "#class" : nozomi.NozomiTagExtractor, + "#pattern" : r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", + "#range" : "1-25", + "#count" : ">= 25", +}, + +{ + "#url" : "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", + "#category": ("", "nozomi", "search"), + "#class" : nozomi.NozomiSearchExtractor, + "#count" : ">= 5", +}, + +) diff --git a/test/results/nsfwalbum.py b/test/results/nsfwalbum.py new file mode 100644 index 00000000..e89b8fb4 --- /dev/null +++ b/test/results/nsfwalbum.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nsfwalbum + + +__tests__ = ( +{ + "#url" : "https://nsfwalbum.com/album/401611", + "#category": ("", "nsfwalbum", "album"), + "#class" : nsfwalbum.NsfwalbumAlbumExtractor, + "#range" : "1-5", + "#sha1_url" : "b0481fc7fad5982da397b6359fbed8421b8ba284", + "#sha1_metadata": "e98f9b0d473c00000831618d0235863b1dd78294", +}, + +) diff --git a/test/results/nudecollect.py b/test/results/nudecollect.py new file mode 100644 index 00000000..423c915f --- /dev/null +++ b/test/results/nudecollect.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nudecollect + + +__tests__ = ( +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/image-4-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "image"), + "#class" : nudecollect.NudecollectImageExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00004-5896498214-43-9689595623/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/9879560327/nudecollect\.com\.jpg", + + "slug" : "20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust", + "title" : "20201220 Teenpornstorage Patritcy Vanessa Lesbian Lust", + "num" : 4, + "count" : 108, + "mirror": 43, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/image-10-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "image"), + "#class" : nudecollect.NudecollectImageExtractor, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px/index-mirror-67-125.html", + "#category": ("", "nudecollect", "album"), + "#class" : nudecollect.NudecollectAlbumExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00\d\d\d-5896498214-67-9689595623/20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px/9879560327/nudecollect\.com\.jpg", + "#count" : 125, + + "slug" : "20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px", + "title" : "20170219 TheWhiteBoxxx Caprice Tracy Loves Hot ass fingering and sensual lesbian sex with alluring Czech babes x125 1080px", + "num" : int, + "mirror": 67, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/page-1-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "album"), + "#class" : nudecollect.NudecollectAlbumExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00\d\d\d-5896498214-43-9689595623/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/9879560327/nudecollect\.com\.jpg", + "#count" : 108, + + "slug" : "20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust", + "title" : "20201220 Teenpornstorage Patritcy Vanessa Lesbian Lust", + "num" : int, + "mirror": 43, +}, + +) diff --git a/test/results/ohpolly.py b/test/results/ohpolly.py new file mode 100644 index 00000000..3ce1968b --- /dev/null +++ b/test/results/ohpolly.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.ohpolly.com/collections/dresses-mini-dresses", + "#category": ("shopify", "ohpolly", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", + "#category": ("shopify", "ohpolly", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/omgmiamiswimwear.py b/test/results/omgmiamiswimwear.py new file mode 100644 index 00000000..27f4d54d --- /dev/null +++ b/test/results/omgmiamiswimwear.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.omgmiamiswimwear.com/collections/fajas", + "#category": ("shopify", "omgmiamiswimwear", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.omgmiamiswimwear.com/products/la-medusa-maxi-dress", + "#category": ("shopify", "omgmiamiswimwear", "product"), + "#class" : shopify.ShopifyProductExtractor, + "#pattern" : r"https://cdn\.shopify\.com/s/files/1/1819/6171/", + "#count" : 5, +}, + +) diff --git a/test/results/paheal.py b/test/results/paheal.py new file mode 100644 index 00000000..3ef0ec58 --- /dev/null +++ b/test/results/paheal.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import paheal + + +__tests__ = ( +{ + "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class" : paheal.PahealTagExtractor, + "#pattern" : r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20", + "#count" : ">= 15", +}, + +{ + "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class" : paheal.PahealTagExtractor, + "#options" : {"metadata": True}, + "#range" : "1", + + "date" : "dt:2018-01-07 07:04:05", + "duration" : 0.0, + "extension" : "jpg", + "filename" : "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "height" : 768, + "id" : 2446128, + "md5" : "b0ceda9d860df1d15b60293a7eb465c1", + "search_tags": "Ayane_Suzuki", + "size" : 205312, + "source" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", + "tags" : "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "uploader" : "XXXname", + "width" : 1024, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/481609", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + "#pattern" : r"https://tulip\.paheal\.net/_images/bbdc1c33410c2cdce7556c7990be26b7/481609%20-%20Azumanga_Daioh%20inanimate%20Osaka%20Vuvuzela\.jpg", + "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", + + "date" : "dt:2010-06-17 15:40:23", + "extension": "jpg", + "file_url" : r"re:https://tulip.paheal.net/_images/bbdc1c33410c", + "filename" : "481609 - Azumanga_Daioh inanimate Osaka Vuvuzela", + "height" : 660, + "id" : 481609, + "md5" : "bbdc1c33410c2cdce7556c7990be26b7", + "size" : 157389, + "source" : "", + "tags" : "Azumanga_Daioh inanimate Osaka Vuvuzela", + "uploader" : "CaptainButtface", + "width" : 614, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/488534", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + + "date" : "dt:2010-06-25 13:51:17", + "height" : 800, + "md5" : "b39edfe455a0381110c710d6ed2ef57d", + "size" : 758989, + "source" : "http://www.furaffinity.net/view/4057821/", + "tags" : "inanimate thelost-dragon Vuvuzela", + "uploader": "leacheate_soup", + "width" : 1200, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/3864982", + "#comment" : "video", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + "#pattern" : r"https://[\w]+\.paheal\.net/_images/7629fc0ff77e32637dde5bf4f992b2cb/3864982%20-%20animated%20Metal_Gear%20Metal_Gear_Solid_V%20Quiet%20Vg_erotica%20webm\.webm", + + "date" : "dt:2020-09-06 01:59:03", + "duration" : 30.0, + "extension": "webm", + "height" : 2500, + "id" : 3864982, + "md5" : "7629fc0ff77e32637dde5bf4f992b2cb", + "size" : 18454938, + "source" : "https://twitter.com/VG_Worklog/status/1302407696294055936", + "tags" : "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", + "uploader" : "justausername", + "width" : 1768, +}, + +) diff --git a/test/results/palanq.py b/test/results/palanq.py new file mode 100644 index 00000000..046e6e88 --- /dev/null +++ b/test/results/palanq.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archive.palanq.win/c/thread/4209598/", + "#category": ("foolfuuka", "palanq", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", +}, + +{ + "#url" : "https://archive.palanq.win/c/", + "#category": ("foolfuuka", "palanq", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archive.palanq.win/_/search/text/test/", + "#category": ("foolfuuka", "palanq", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archive.palanq.win/c/gallery", + "#category": ("foolfuuka", "palanq", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/patreon.py b/test/results/patreon.py new file mode 100644 index 00000000..adad12e3 --- /dev/null +++ b/test/results/patreon.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import patreon +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.patreon.com/koveliana", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#range" : "1-25", + "#count" : ">= 25", + + "attachments" : list, + "comment_count": int, + "content" : str, + "creator" : dict, + "date" : datetime.datetime, + "id" : int, + "images" : list, + "like_count" : int, + "post_type" : str, + "published_at" : str, + "title" : str, +}, + +{ + "#url" : "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#count" : 1, + + "date": "dt:2020-03-30 21:21:44", +}, + +{ + "#url" : "https://www.patreon.com/kovelianot", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.patreon.com/user?u=2931440", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, +}, + +{ + "#url" : "https://www.patreon.com/user/posts/?u=2931440", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, +}, + +{ + "#url" : "https://www.patreon.com/home", + "#category": ("", "patreon", "user"), + "#class" : patreon.PatreonUserExtractor, +}, + +{ + "#url" : "https://www.patreon.com/posts/precious-metal-23563293", + "#comment" : "postfile + attachments", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://www.patreon.com/posts/56127163", + "#comment" : "postfile + content", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#count" : 3, + + "filename": r"re:^(?!1).+$", +}, + +{ + "#url" : "https://www.patreon.com/posts/free-post-12497641", + "#comment" : "tags (#1539)", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + + "tags": ["AWMedia"], +}, + +{ + "#url" : "https://www.patreon.com/posts/not-found-123", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/pawoo.py b/test/results/pawoo.py new file mode 100644 index 00000000..5a9bfcaa --- /dev/null +++ b/test/results/pawoo.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://pawoo.net/@yoru_nine/", + "#category": ("mastodon", "pawoo", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#range" : "1-60", + "#count" : 60, +}, + +{ + "#url" : "https://pawoo.net/bookmarks", + "#category": ("mastodon", "pawoo", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://pawoo.net/users/yoru_nine/following", + "#category": ("mastodon", "pawoo", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://pawoo.net/@yoru_nine/105038878897832922", + "#category": ("mastodon", "pawoo", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", +}, + +) diff --git a/test/results/photobucket.py b/test/results/photobucket.py new file mode 100644 index 00000000..90b5b83f --- /dev/null +++ b/test/results/photobucket.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import photobucket + + +__tests__ = ( +{ + "#url" : "https://s369.photobucket.com/user/CrpyLrkr/library", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#pattern" : r"https?://[oi]+\d+.photobucket.com/albums/oo139/", + "#count" : ">= 50", +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/library/", + "#comment" : "subalbums of main 'directory'", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#options" : {"image-filter": "False"}, + "#pattern" : photobucket.PhotobucketAlbumExtractor.pattern, + "#count" : 1, +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/library/Basketball", + "#comment" : "subalbums of subalbum without images", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#pattern" : photobucket.PhotobucketAlbumExtractor.pattern, + "#count" : ">= 9", +}, + +{ + "#url" : "https://s1277.photobucket.com/user/sinisterkat44/library/", + "#comment" : "private (missing JSON data)", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://s1110.photobucket.com/user/chndrmhn100/library/Chandu%20is%20the%20King?sort=3&page=1", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/media/Untitled-3-1.jpg.html", + "#category": ("", "photobucket", "image"), + "#class" : photobucket.PhotobucketImageExtractor, + "#sha1_url" : "3b647deeaffc184cc48c89945f67574559c9051f", + "#sha1_metadata": "69732741b2b351db7ecaa77ace2fdb39f08ca5a3", +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/media/IsotopeswBros.jpg.html?sort=3&o=2", + "#category": ("", "photobucket", "image"), + "#class" : photobucket.PhotobucketImageExtractor, + "#sha1_url" : "12c1890c09c9cdb8a88fba7eec13f324796a8d7b", + "#sha1_metadata": "61200a223df6c06f45ac3d30c88b3f5b048ce9a8", +}, + +) diff --git a/test/results/photovogue.py b/test/results/photovogue.py new file mode 100644 index 00000000..6898f5d9 --- /dev/null +++ b/test/results/photovogue.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import photovogue +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class" : photovogue.PhotovogueUserExtractor, +}, + +{ + "#url" : "https://vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class" : photovogue.PhotovogueUserExtractor, + "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", + + "date" : datetime.datetime, + "favorite_count" : int, + "favorited" : list, + "id" : int, + "image_id" : str, + "is_favorite" : False, + "orientation" : r"re:portrait|landscape", + "photographer" : { + "biography" : "Born in 1995. Live in Bologna.", + "city" : "Bologna", + "country_id" : 106, + "favoritedCount": int, + "id" : 221252, + "isGold" : bool, + "isPro" : bool, + "latitude" : str, + "longitude" : str, + "name" : "Arianna Mattarozzi", + "user_id" : "38cb0601-4a85-453c-b7dc-7650a037f2ab", + "websites" : list, + }, + "photographer_id": 221252, + "tags" : list, + "title" : str, +}, + +) diff --git a/test/results/picarto.py b/test/results/picarto.py new file mode 100644 index 00000000..07456260 --- /dev/null +++ b/test/results/picarto.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import picarto +import datetime + + +__tests__ = ( +{ + "#url" : "https://picarto.tv/fnook/gallery/default/", + "#category": ("", "picarto", "gallery"), + "#class" : picarto.PicartoGalleryExtractor, + "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", + "#count" : ">= 7", + + "date": datetime.datetime, +}, + +) diff --git a/test/results/piczel.py b/test/results/piczel.py new file mode 100644 index 00000000..84d1d01b --- /dev/null +++ b/test/results/piczel.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import piczel + + +__tests__ = ( +{ + "#url" : "https://piczel.tv/gallery/Bikupan", + "#category": ("", "piczel", "user"), + "#class" : piczel.PiczelUserExtractor, + "#range" : "1-100", + "#count" : ">= 100", +}, + +{ + "#url" : "https://piczel.tv/gallery/Lulena/1114", + "#category": ("", "piczel", "folder"), + "#class" : piczel.PiczelFolderExtractor, + "#count" : ">= 4", +}, + +{ + "#url" : "https://piczel.tv/gallery/image/7807", + "#category": ("", "piczel", "image"), + "#class" : piczel.PiczelImageExtractor, + "#pattern" : r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", + "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", + + "created_at" : "2018-07-22T05:13:58.000Z", + "date" : "dt:2018-07-22 05:13:58", + "description" : None, + "extension" : "png", + "favorites_count" : int, + "folder_id" : 1113, + "id" : 7807, + "is_flash" : False, + "is_video" : False, + "multi" : False, + "nsfw" : False, + "num" : 0, + "password_protected": False, + "tags" : [ + "fanart", + "commission", + "altair", + "recreators", + ], + "title" : "Altair", + "user" : dict, + "views" : int, +}, + +) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py new file mode 100644 index 00000000..a7c73beb --- /dev/null +++ b/test/results/pillowfort.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pillowfort +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.pillowfort.social/posts/27510", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#pattern" : r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", + "#count" : 4, + + "avatar_url" : str, + "col" : 0, + "commentable" : True, + "comments_count" : int, + "community_id" : None, + "content" : str, + "created_at" : str, + "date" : datetime.datetime, + "deleted" : None, + "deleted_at" : None, + "deleted_by_mod" : None, + "deleted_for_flag_id": None, + "embed_code" : None, + "id" : int, + "last_activity" : str, + "last_activity_elapsed": str, + "last_edited_at" : str, + "likes_count" : int, + "media_type" : "picture", + "nsfw" : False, + "num" : int, + "original_post_id": None, + "original_post_user_id": None, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id" : 27510, + "post_type" : "picture", + "privacy" : "public", + "reblog_copy_info": list, + "rebloggable" : True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count" : int, + "row" : int, + "small_image_url" : None, + "tags" : list, + "time_elapsed" : str, + "timestamp" : str, + "title" : "What is Pillowfort.social?", + "updated_at" : str, + "url" : r"re:https://img3.pillowfort.social/posts/.*\.png", + "user_id" : 5, + "username" : "Staff", +}, + +{ + "#url" : "https://www.pillowfort.social/posts/1557500", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#options" : { + "external": True, + "inline" : False, + }, + "#pattern" : r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", +}, + +{ + "#url" : "https://www.pillowfort.social/posts/1672518", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#options" : {"inline": True}, + "#count" : 3, +}, + +{ + "#url" : "https://www.pillowfort.social/Pome", + "#category": ("", "pillowfort", "user"), + "#class" : pillowfort.PillowfortUserExtractor, + "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", + "#range" : "1-15", + "#count" : 15, +}, + +) diff --git a/test/results/pinterest.py b/test/results/pinterest.py new file mode 100644 index 00000000..26671bff --- /dev/null +++ b/test/results/pinterest.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pinterest +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pinterest.com/pin/858146903966145189/", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#sha1_url" : "afb3c26719e3a530bb0e871c480882a801a4e8a5", + "#sha1_content": [ + "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", + "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", + ], +}, + +{ + "#url" : "https://www.pinterest.com/pin/422564377542934214/", + "#comment" : "video pin (#1189)", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#pattern" : r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", +}, + +{ + "#url" : "https://www.pinterest.com/pin/858146903966145188/", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/test-/", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/", + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/stuff/", + "#comment" : "board with sections (#835)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#options" : {"sections": True}, + "#count" : 4, +}, + +{ + "#url" : "https://www.pinterest.de/g1952849/secret/", + "#comment" : "secret board (#1055)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/g1952848/test/", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#exception": exception.GalleryDLException, +}, + +{ + "#url" : "https://www.pinterest.co.uk/hextra7519/based-animals/", + "#comment" : ".co.uk TLD (#914)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/", + "#category": ("", "pinterest", "user"), + "#class" : pinterest.PinterestUserExtractor, + "#pattern" : pinterest.PinterestBoardExtractor.pattern, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/_saved/", + "#category": ("", "pinterest", "user"), + "#class" : pinterest.PinterestUserExtractor, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/pins/", + "#category": ("", "pinterest", "allpins"), + "#class" : pinterest.PinterestAllpinsExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", + "#count" : 7, +}, + +{ + "#url" : "https://www.pinterest.de/digitalmomblog/_created/", + "#category": ("", "pinterest", "created"), + "#class" : pinterest.PinterestCreatedExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/stuff/section", + "#category": ("", "pinterest", "section"), + "#class" : pinterest.PinterestSectionExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/search/pins/?q=nature", + "#category": ("", "pinterest", "search"), + "#class" : pinterest.PinterestSearchExtractor, + "#range" : "1-50", + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.pinterest.com/pin/858146903966145189/#related", + "#category": ("", "pinterest", "related-pin"), + "#class" : pinterest.PinterestRelatedPinExtractor, + "#range" : "31-70", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/test-/#related", + "#category": ("", "pinterest", "related-board"), + "#class" : pinterest.PinterestRelatedBoardExtractor, + "#range" : "31-70", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://pin.it/Hvt8hgT", + "#category": ("", "pinterest", "pinit"), + "#class" : pinterest.PinterestPinitExtractor, + "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", +}, + +{ + "#url" : "https://pin.it/Hvt8hgS", + "#category": ("", "pinterest", "pinit"), + "#class" : pinterest.PinterestPinitExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/pinupgirlclothing.py b/test/results/pinupgirlclothing.py new file mode 100644 index 00000000..ac82ad53 --- /dev/null +++ b/test/results/pinupgirlclothing.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://pinupgirlclothing.com/collections/evening", + "#category": ("shopify", "pinupgirlclothing", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", + "#category": ("shopify", "pinupgirlclothing", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/pixhost.py b/test/results/pixhost.py new file mode 100644 index 00000000..82fbea98 --- /dev/null +++ b/test/results/pixhost.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://pixhost.to/show/190/130327671_test-.png", + "#category": ("imagehost", "pixhost", "image"), + "#class" : imagehosts.PixhostImageExtractor, + "#sha1_url" : "4e5470dcf6513944773044d40d883221bbc46cff", + "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://pixhost.to/gallery/jSMFq", + "#category": ("imagehost", "pixhost", "gallery"), + "#class" : imagehosts.PixhostGalleryExtractor, + "#pattern" : imagehosts.PixhostImageExtractor.pattern, + "#count" : 3, +}, + +) diff --git a/test/results/pixiv.py b/test/results/pixiv.py new file mode 100644 index 00000000..34c44f8c --- /dev/null +++ b/test/results/pixiv.py @@ -0,0 +1,520 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pixiv +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pixiv.net/en/users/173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/u/173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/member.php?id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/mypage.php#id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/#id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/artworks", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", + "#comment" : "illusts with specific tag", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?id=173531", + "#comment" : "deleted account", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#options" : {"metadata": True}, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/manga", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/illustrations", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/avatar", + "#category": ("", "pixiv", "avatar"), + "#class" : pixiv.PixivAvatarExtractor, + "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/194921/background", + "#category": ("", "pixiv", "background"), + "#class" : pixiv.PixivBackgroundExtractor, + "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", +}, + +{ + "#url" : "https://pixiv.me/del_shannon", + "#category": ("", "pixiv", "me"), + "#class" : pixiv.PixivMeExtractor, + "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", +}, + +{ + "#url" : "https://pixiv.me/del_shanno", + "#category": ("", "pixiv", "me"), + "#class" : pixiv.PixivMeExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/artworks/966412", + "#comment" : "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#sha1_url" : "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", + + "date" : "dt:2008-06-12 15:29:13", + "date_url": "dt:2008-06-12 15:29:13", +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", + "#comment" : "ugoira", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#sha1_url": "7267695a985c4db8759bebcf8d21dbdd2d2317ef", + + "frames" : list, + "date" : "dt:2018-01-14 15:06:08", + "date_url": "dt:2018-01-15 04:24:48", +}, + +{ + "#url" : "https://www.pixiv.net/artworks/966412", + "#comment" : "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#options" : {"related": True}, + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.pixiv.net/en/artworks/966412", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/i/966412", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://img.pixiv.net/img/soundcross/42626136.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/bookmarks/artworks", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=173530", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", + "#comment" : "bookmarks with specific tag", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", + "#comment" : "bookmarks with specific tag (legacy url)", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php", + "#comment" : "own bookmarks", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, + "#options" : {"metadata-bookmark": True}, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + + "tags_bookmark": [ + "47", + "hitman", + ], +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?tag=foobar", + "#comment" : "own bookmarks with tag (#596)", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/following", + "#comment" : "followed users (#515)", + "#category": ("", "pixiv", "following"), + "#class" : pixiv.PixivFavoriteExtractor, + "#pattern" : pixiv.PixivUserExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=173530&type=user", + "#comment" : "followed users (legacy url) (#515)", + "#category": ("", "pixiv", "following"), + "#class" : pixiv.PixivFavoriteExtractor, + "#pattern" : pixiv.PixivUserExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark.php?id=173530", + "#comment" : "touch URLs", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark.php", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php?mode=unknown", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/en/tags/Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/search.php?word=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class" : pixiv.PixivFollowExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class" : pixiv.PixivFollowExtractor, +}, + +{ + "#url" : "https://www.pixivision.net/en/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class" : pixiv.PixivPixivisionExtractor, +}, + +{ + "#url" : "https://pixivision.net/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class" : pixiv.PixivPixivisionExtractor, + "#count" : 7, + + "pixivision_id" : "2791", + "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", +}, + +{ + "#url" : "https://www.pixiv.net/user/10509347/series/21859", + "#category": ("", "pixiv", "series"), + "#class" : pixiv.PixivSeriesExtractor, + "#range" : "1-10", + "#count" : 10, + + "num_series": int, + "series" : { + "canonical" : "https://www.pixiv.net/user/10509347/series/21859", + "description": str, + "ogp" : dict, + "title" : "先輩がうざい後輩の話", + "total" : int, + "twitter" : dict, + }, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=19612040", + "#comment" : "full series", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#count" : 1, + "#sha1_content": "8c818474153cbd2f221ee08766e1d634c821d8b4", + + "caption" : r"re:「無能な名無し」と呼ばれ虐げられて育った鈴\(すず\)は、", + "comment_access_control": 0, + "create_date" : "2023-04-02T15:18:58+09:00", + "date" : "dt:2023-04-02 06:18:58", + "id" : 19612040, + "is_bookmarked" : False, + "is_muted" : False, + "is_mypixiv_only" : False, + "is_original" : True, + "is_x_restricted" : False, + "novel_ai_type" : 1, + "page_count" : 1, + "rating" : "General", + "restrict" : 0, + "series" : { + "id" : 10278364, + "title": "龍の贄嫁〜無能な名無しと虐げられていましたが、どうやら異母妹に霊力を搾取されていたようです〜", + }, + "tags" : [ + "和風ファンタジー", + "溺愛", + "神様", + "ヤンデレ", + "執着", + "異能", + "ざまぁ", + "学園", + "神嫁", + ], + "text_length" : 5974, + "title" : "異母妹から「無能な名無し」と虐げられていた私、どうやら異母妹に霊力を搾取されていたようです（１）", + "user" : { + "account": "yukinaga_chifuyu", + "id" : 77055466, + }, + "visible" : True, + "x_restrict" : 0, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", + "#comment" : "embeds", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#options" : {"embeds": True}, + "#count" : 3, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=19612040", + "#comment" : "full series", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#options" : {"full-series": True}, + "#count" : 4, +}, + +{ + "#url" : "https://www.pixiv.net/n/19612040", + "#comment" : "short URL", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/novels", + "#category": ("", "pixiv", "novel-user"), + "#class" : pixiv.PixivNovelUserExtractor, + "#pattern" : "^text:", + "#range" : "1-5", + "#count" : 5, +}, + +{ + "#url" : "https://www.pixiv.net/novel/series/10278364", + "#category": ("", "pixiv", "novel-series"), + "#class" : pixiv.PixivNovelSeriesExtractor, + "#count" : 4, + "#sha1_content": "b06abed001b3f6ccfb1579699e9a238b46d38ea2", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", + "#category": ("", "pixiv", "novel-bookmark"), + "#class" : pixiv.PixivNovelBookmarkExtractor, + "#count" : 1, + "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", + "#category": ("", "pixiv", "novel-bookmark"), + "#class" : pixiv.PixivNovelBookmarkExtractor, +}, + +{ + "#url" : "https://sketch.pixiv.net/@nicoby", + "#category": ("", "pixiv", "sketch"), + "#class" : pixiv.PixivSketchExtractor, + "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", + "#count" : ">= 35", +}, + +) diff --git a/test/results/pixnet.py b/test/results/pixnet.py new file mode 100644 index 00000000..9c086526 --- /dev/null +++ b/test/results/pixnet.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pixnet + + +__tests__ = ( +{ + "#url" : "https://albertayu773.pixnet.net/album/photo/159443828", + "#category": ("", "pixnet", "image"), + "#class" : pixnet.PixnetImageExtractor, + "#sha1_url" : "156564c422138914c9fa5b42191677b45c414af4", + "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", + "#sha1_content" : "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/set/15078995", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#sha1_url" : "6535712801af47af51110542f4938a7cef44557f", + "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", +}, + +{ + "#url" : "https://anrine910070.pixnet.net/album/set/5917493", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#sha1_url" : "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", + "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", +}, + +{ + "#url" : "https://sky92100.pixnet.net/album/set/17492544", + "#comment" : "password-protected", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/folder/1405768", + "#category": ("", "pixnet", "folder"), + "#class" : pixnet.PixnetFolderExtractor, + "#pattern" : pixnet.PixnetSetExtractor.pattern, + "#count" : ">= 15", +}, + +{ + "#url" : "https://albertayu773.pixnet.net/", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/blog", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, + "#pattern" : pixnet.PixnetFolderExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://anrine910070.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, + "#pattern" : pixnet.PixnetSetExtractor.pattern, + "#count" : ">= 14", +}, + +) diff --git a/test/results/plurk.py b/test/results/plurk.py new file mode 100644 index 00000000..11600bae --- /dev/null +++ b/test/results/plurk.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import plurk + + +__tests__ = ( +{ + "#url" : "https://www.plurk.com/plurkapi", + "#category": ("", "plurk", "timeline"), + "#class" : plurk.PlurkTimelineExtractor, + "#pattern" : "https?://.+", + "#count" : ">= 23", +}, + +{ + "#url" : "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class" : plurk.PlurkPostExtractor, + "#count" : 3, + "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", +}, + +{ + "#url" : "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class" : plurk.PlurkPostExtractor, + "#options" : {"comments": True}, + "#count" : ">= 210", +}, + +) diff --git a/test/results/poipiku.py b/test/results/poipiku.py new file mode 100644 index 00000000..08658bae --- /dev/null +++ b/test/results/poipiku.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import poipiku + + +__tests__ = ( +{ + "#url" : "https://poipiku.com/25049/", + "#category": ("", "poipiku", "user"), + "#class" : poipiku.PoipikuUserExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", + "#category": ("", "poipiku", "user"), + "#class" : poipiku.PoipikuUserExtractor, +}, + +{ + "#url" : "https://poipiku.com/25049/5864576.html", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", + + "count" : "1", + "description" : "", + "extension" : "png", + "filename" : "005864576_EWN1Y65gQ", + "num" : 1, + "post_category": "DOODLE", + "post_id" : "5864576", + "user_id" : "25049", + "user_name" : "ユキウサギ", +}, + +{ + "#url" : "https://poipiku.com/2166245/6411749.html", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", + "#count" : 4, + + "count" : "4", + "description" : "絵茶の産物ネタバレあるやつ", + "num" : int, + "post_category": "SPOILER", + "post_id" : "6411749", + "user_id" : "2166245", + "user_name" : "wadahito", +}, + +{ + "#url" : "https://poipiku.com/3572553/5776587.html", + "#comment" : "different warning button style", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", + "#count" : 3, + + "count" : "3", + "description" : "ORANGE OASISボスネタバレ<br />曲も大好き<br />2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", + "num" : int, + "post_category": "SPOILER", + "post_id" : "5776587", + "user_id" : "3572553", + "user_name" : "nagakun", +}, + +) diff --git a/test/results/ponybooru.py b/test/results/ponybooru.py new file mode 100644 index 00000000..6955bb49 --- /dev/null +++ b/test/results/ponybooru.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", +}, + +{ + "#url" : "https://www.ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://ponybooru.org/search?q=cute", + "#category": ("philomena", "ponybooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://ponybooru.org/galleries/27", + "#category": ("philomena", "ponybooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#count" : ">= 24", +}, + +) diff --git a/test/results/pornhub.py b/test/results/pornhub.py new file mode 100644 index 00000000..e7aaf8da --- /dev/null +++ b/test/results/pornhub.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pornhub +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pornhub.com/album/19289801", + "#category": ("", "pornhub", "gallery"), + "#class" : pornhub.PornhubGalleryExtractor, + "#pattern" : r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", + "#count" : ">= 300", + + "id" : int, + "num" : int, + "score" : int, + "views" : int, + "caption": str, + "user" : "Danika Mori", + "gallery": { + "id" : 19289801, + "score": int, + "views": int, + "tags" : list, + "title": "Danika Mori Best Moments", + }, +}, + +{ + "#url" : "https://www.pornhub.com/album/69040172", + "#category": ("", "pornhub", "gallery"), + "#class" : pornhub.PornhubGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://www.pornhub.com/gif/33643461", + "#category": ("", "pornhub", "gif"), + "#class" : pornhub.PornhubGifExtractor, + "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/033/643/461/33643461a\.webm", + + "date" : "dt:2020-10-31 00:00:00", + "extension": "webm", + "filename" : "33643461a", + "id" : "33643461", + "tags" : [ + "big boobs", + "lana rhoades", + ], + "title" : "Big boobs", + "url" : str, + "user" : "Lana Rhoades", +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori", + "#category": ("", "pornhub", "user"), + "#class" : pornhub.PornhubUserExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori/photos", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, + "#pattern" : pornhub.PornhubGalleryExtractor.pattern, + "#count" : ">= 6", +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/public", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/private", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/favorites", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/model/bossgirl/photos", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori/gifs", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, + "#pattern" : pornhub.PornhubGifExtractor.pattern, + "#count" : ">= 42", +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/gifs", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/model/bossgirl/gifs/video", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, +}, + +) diff --git a/test/results/pornpics.py b/test/results/pornpics.py new file mode 100644 index 00000000..91a10ca0 --- /dev/null +++ b/test/results/pornpics.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pornpics + + +__tests__ = ( +{ + "#url" : "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", + "#category": ("", "pornpics", "gallery"), + "#class" : pornpics.PornpicsGalleryExtractor, + "#pattern" : r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", + + "categories": [ + "MILF", + "Amateur", + "Sexy", + "Outdoor", + ], + "channel" : "FTV MILFs", + "count" : 17, + "gallery_id": 62610699, + "models" : ["Danielle"], + "num" : int, + "slug" : "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + "tags" : [ + "Amateur MILF", + "Sexy MILF", + ], + "title" : "British beauty Danielle flashes hot breasts, ass and snatch in the forest", + "views" : int, +}, + +{ + "#url" : "https://pornpics.com/es/galleries/62610699", + "#category": ("", "pornpics", "gallery"), + "#class" : pornpics.PornpicsGalleryExtractor, + + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", +}, + +{ + "#url" : "https://www.pornpics.com/tags/summer-dress/", + "#category": ("", "pornpics", "tag"), + "#class" : pornpics.PornpicsTagExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://pornpics.com/fr/tags/summer-dress", + "#category": ("", "pornpics", "tag"), + "#class" : pornpics.PornpicsTagExtractor, +}, + +{ + "#url" : "https://www.pornpics.com/?q=nature", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.pornpics.com/channels/femjoy/", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.pornpics.com/pornstars/emma-brown/", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://pornpics.com/jp/?q=nature", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +{ + "#url" : "https://pornpics.com/it/channels/femjoy", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +{ + "#url" : "https://pornpics.com/pt/pornstars/emma-brown", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +) diff --git a/test/results/pornreactor.py b/test/results/pornreactor.py new file mode 100644 index 00000000..0062d19a --- /dev/null +++ b/test/results/pornreactor.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://pornreactor.cc/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#range" : "1-25", + "#count" : ">= 25", +}, + +{ + "#url" : "http://fapreactor.com/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/search?q=ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://fapreactor.com/search/ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class" : reactor.ReactorUserExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://fapreactor.com/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", + "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", +}, + +{ + "#url" : "http://fapreactor.com/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", +}, + +) diff --git a/test/results/postimg.py b/test/results/postimg.py new file mode 100644 index 00000000..ba4b8a86 --- /dev/null +++ b/test/results/postimg.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://www.postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://pixxxels.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://postimg.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, + "#sha1_url" : "72f3c8b1d6c6601a20ad58f35635494b4891a99e", + "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", + "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", +}, + +{ + "#url" : "https://postimg.cc/gallery/wxpDLgX", + "#category": ("imagehost", "postimg", "gallery"), + "#class" : imagehosts.PostimgGalleryExtractor, + "#pattern" : imagehosts.PostimgImageExtractor.pattern, + "#count" : 22, +}, + +) diff --git a/test/results/powermanga.py b/test/results/powermanga.py new file mode 100644 index 00000000..fb02b0cc --- /dev/null +++ b/test/results/powermanga.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolslide + + +__tests__ = ( +{ + "#url" : "https://read.powermanga.org/read/one_piece_digital_colour_comics/en/0/75/", + "#category": ("foolslide", "powermanga", "chapter"), + "#class" : foolslide.FoolslideChapterExtractor, + "#sha1_url" : "854c5817f8f767e1bccd05fa9d58ffb5a4b09384", + "#sha1_metadata": "a60c42f2634b7387899299d411ff494ed0ad6dbe", +}, + +{ + "#url" : "https://read.powermanga.org/series/one_piece_digital_colour_comics/", + "#category": ("foolslide", "powermanga", "manga"), + "#class" : foolslide.FoolslideMangaExtractor, + "#count" : ">= 1", + + "chapter" : int, + "chapter_minor" : str, + "chapter_string": str, + "group" : "PowerManga", + "lang" : "en", + "language" : "English", + "manga" : "One Piece Digital Colour Comics", + "title" : str, + "volume" : int, +}, + +) diff --git a/test/results/pururin.py b/test/results/pururin.py new file mode 100644 index 00000000..e57c7fe7 --- /dev/null +++ b/test/results/pururin.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pururin + + +__tests__ = ( +{ + "#url" : "https://pururin.to/gallery/38661/iowant-2", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, + "#pattern" : r"https://i\.pururin\.to/38661/\d+\.jpg", + + "title" : r"re:I ?owant 2!!", + "title_en" : r"re:I ?owant 2!!", + "title_ja" : "", + "gallery_id": 38661, + "count" : 19, + "artist" : ["Shoda Norihiro"], + "group" : ["Obsidian Order"], + "parody" : ["Kantai Collection"], + "characters": [ + "Iowa", + "Teitoku", + ], + "tags" : list, + "type" : "Doujinshi", + "collection": ["I owant you!"], + "convention": ["C92"], + "rating" : float, + "uploader" : "demo", + "scanlator" : [ + "mrwayne", + "The Lost Light", + ], + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://pururin.to/gallery/7661/unisis-team-vanilla", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, + "#count" : 17, +}, + +{ + "#url" : "https://pururin.io/gallery/38661/iowant-2", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, +}, + +) diff --git a/test/results/raidlondon.py b/test/results/raidlondon.py new file mode 100644 index 00000000..93aab265 --- /dev/null +++ b/test/results/raidlondon.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.raidlondon.com/collections/flats", + "#category": ("shopify", "raidlondon", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", + "#category": ("shopify", "raidlondon", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/rbt.py b/test/results/rbt.py new file mode 100644 index 00000000..5a81ad1a --- /dev/null +++ b/test/results/rbt.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://rbt.asia/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", +}, + +{ + "#url" : "https://archive.rebeccablacktech.com/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", +}, + +{ + "#url" : "https://rbt.asia/g/", + "#category": ("foolfuuka", "rbt", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://rbt.asia/_/search/text/test/", + "#category": ("foolfuuka", "rbt", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://rbt.asia/g/gallery/8", + "#category": ("foolfuuka", "rbt", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/reactor.py b/test/results/reactor.py new file mode 100644 index 00000000..7ff6dcce --- /dev/null +++ b/test/results/reactor.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://reactor.cc/tag/gif", + "#category": ("reactor", "reactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://reactor.cc/search?q=Art", + "#category": ("reactor", "reactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://reactor.cc/user/Dioklet", + "#category": ("reactor", "reactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://reactor.cc/post/4999736", + "#category": ("reactor", "reactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", +}, + +) diff --git a/test/results/readcomiconline.py b/test/results/readcomiconline.py new file mode 100644 index 00000000..883ed321 --- /dev/null +++ b/test/results/readcomiconline.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import readcomiconline + + +__tests__ = ( +{ + "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", + "#category": ("", "readcomiconline", "issue"), + "#class" : readcomiconline.ReadcomiconlineIssueExtractor, + "#pattern" : r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", + "#count" : 36, + "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", +}, + +{ + "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h", + "#category": ("", "readcomiconline", "comic"), + "#class" : readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url" : "74eb8b9504b4084fcc9367b341300b2c52260918", + "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", +}, + +{ + "#url" : "https://readcomiconline.to/Comic/Bazooka-Jules", + "#category": ("", "readcomiconline", "comic"), + "#class" : readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url" : "2f66a467a772df4d4592e97a059ddbc3e8991799", + "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", +}, + +) diff --git a/test/results/realbooru.py b/test/results/realbooru.py new file mode 100644 index 00000000..e99634b4 --- /dev/null +++ b/test/results/realbooru.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://realbooru.com/index.php?page=post&s=list&tags=wine", + "#category": ("gelbooru_v02", "realbooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : ">= 64", +}, + +{ + "#url" : "https://realbooru.com/index.php?page=pool&s=show&id=1", + "#category": ("gelbooru_v02", "realbooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://realbooru.com/index.php?page=favorites&s=view&id=274", + "#category": ("gelbooru_v02", "realbooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://realbooru.com/index.php?page=post&s=view&id=668483", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : {"tags": True}, + "#pattern" : r"https://realbooru\.com//?images/dc/b5/dcb5c0ce9ec0bf74a6930608985f4719\.jpeg", + "#sha1_content": "7f5873ce3b6cd295ea2e81fcb49583098ea9c8da", + + "tags_general": "1girl blonde blonde_hair blue_eyes cute female female_only looking_at_viewer smile solo solo_female teeth", + "tags_model" : "jennifer_lawrence", +}, + +) diff --git a/test/results/recursive.py b/test/results/recursive.py new file mode 100644 index 00000000..2ad31dd4 --- /dev/null +++ b/test/results/recursive.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import recursive + + +__tests__ = ( +{ + "#url" : "recursive:https://pastebin.com/raw/FLwrCYsT", + "#category": ("", "recursive", ""), + "#class" : recursive.RecursiveExtractor, + "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", +}, + +) diff --git a/test/results/reddit.py b/test/results/reddit.py new file mode 100644 index 00000000..330ef5b6 --- /dev/null +++ b/test/results/reddit.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reddit + + +__tests__ = ( +{ + "#url" : "https://www.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, + "#range" : "1-20", + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", + "#category": ("", "reddit", "subreddit-top"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://old.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://np.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://m.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://www.reddit.com/", + "#category": ("", "reddit", "home"), + "#class" : reddit.RedditHomeExtractor, + "#range" : "1-20", + "#count" : ">= 20", +}, + +{ + "#url" : "https://old.reddit.com/top/?sort=top&t=month", + "#category": ("", "reddit", "home-top"), + "#class" : reddit.RedditHomeExtractor, +}, + +{ + "#url" : "https://www.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.reddit.com/user/username/gilded/?sort=top&t=month", + "#category": ("", "reddit", "user-gilded"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://old.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://www.reddit.com/u/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"comments": 500}, + "#pattern" : "https://", + "#count" : 3, +}, + +{ + "#url" : "https://www.reddit.com/gallery/hrrh23", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 3, + "#sha1_url" : "25b91ede15459470274dd17291424b037ed8b0ae", + "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "ytdl:https://v.redd.it/gyh95hiqc0b11", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"videos": "ytdl"}, + "#pattern" : "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"videos": "dash"}, + "#pattern" : r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/gallery/icfgzv", + "#comment" : "deleted gallery (#953)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.reddit.com/r/araragi/comments/ib32hm", + "#comment" : "animated gallery items (#955)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"https://i\.redd\.it/\w+\.gif", + "#count" : 2, +}, + +{ + "#url" : "https://www.reddit.com/r/cosplay/comments/jvwaqr", + "#comment" : "'failed' gallery item (#1127)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/kpopfap/comments/qjj04q/", + "#comment" : "gallery with no 'media_metadata' (#2001)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", + "#comment" : "user page submission (#2301)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "https://i.redd.it/8fpgv17yqlh81.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/kittengifs/comments/12m0b8d", + "#comment" : "cross-posted video (#887, #3586, #3976)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"ytdl:https://v\.redd\.it/cvabpjacrvta1", +}, + +{ + "#url" : "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", + "#comment" : "preview.redd.it (#4470)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", +}, + +{ + "#url" : "https://old.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://np.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://m.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://redd.it/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://i.redd.it/upjtjcx2npzz.jpg", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#sha1_url" : "0de614900feef103e580b632190458c0b62b641a", + "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", +}, + +{ + "#url" : "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#sha1_url" : "f24f25efcedaddeec802e46c60d77ef975dc52a5", + "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", +}, + +{ + "#url" : "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", + "#comment" : "preview.redd.it -> i.redd.it", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#pattern" : r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", +}, + +) diff --git a/test/results/redgifs.py b/test/results/redgifs.py new file mode 100644 index 00000000..2ae76485 --- /dev/null +++ b/test/results/redgifs.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import redgifs + + +__tests__ = ( +{ + "#url" : "https://www.redgifs.com/users/Natalifiction", + "#category": ("", "redgifs", "user"), + "#class" : redgifs.RedgifsUserExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#count" : ">= 100", +}, + +{ + "#url" : "https://v3.redgifs.com/users/lamsinka89", + "#category": ("", "redgifs", "user"), + "#class" : redgifs.RedgifsUserExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#count" : ">= 100", +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections/2631326bbd", + "#category": ("", "redgifs", "collection"), + "#class" : redgifs.RedgifsCollectionExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", + "#category": ("", "redgifs", "collection"), + "#class" : redgifs.RedgifsCollectionExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections", + "#category": ("", "redgifs", "collections"), + "#class" : redgifs.RedgifsCollectionsExtractor, + "#pattern" : r"https://www\.redgifs\.com/users/boombah123/collections/\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://www.redgifs.com/niches/boobs", + "#category": ("", "redgifs", "niches"), + "#class" : redgifs.RedgifsNichesExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/niches/ass", + "#category": ("", "redgifs", "niches"), + "#class" : redgifs.RedgifsNichesExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/gifs/jav", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.redgifs.com/gifs/jav?order=best&verified=1", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://www.redgifs.com/browse?type=i&verified=y&order=top7", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://v3.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://redgifs.com/watch/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", + "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", +}, + +{ + "#url" : "https://www.redgifs.com/watch/desertedbaregraywolf", + "#comment" : "gallery (#4021)", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", + "#count" : 4, + + "num" : int, + "count" : 4, + "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", +}, + +{ + "#url" : "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +) diff --git a/test/results/rule34.py b/test/results/rule34.py new file mode 100644 index 00000000..ca90e511 --- /dev/null +++ b/test/results/rule34.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", + "#category": ("gelbooru_v02", "rule34", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#pattern" : r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", + "#count" : 2, + "#sha1_content": [ + "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "622e80be3f496672c44aab5c47fbc6941c61bc79", + ], +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=pool&s=show&id=179", + "#category": ("gelbooru_v02", "rule34", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", + "#category": ("gelbooru_v02", "rule34", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=post&s=view&id=863", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : { + "tags" : True, + "notes": True, + }, + "#pattern" : r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", + "#sha1_content": [ + "a43f418aa350039af0d11cae501396a33bbe2201", + "67b516295950867e1c1ab6bc13b35d3b762ed2a3", + ], + + "tags_artist" : "reverse_noise yamu_(reverse_noise)", + "tags_character": "hong_meiling", + "tags_copyright": "touhou", + "tags_general" : str, + "tags_metadata" : "censored translated", + "notes" : [ + { + "body" : "It feels angry, I'm losing myself... It won't calm down!", + "height": 65, + "id" : 93586, + "width" : 116, + "x" : 22, + "y" : 333, + }, + { + "body" : "REPUTATION OF RAGE", + "height": 272, + "id" : 93587, + "width" : 199, + "x" : 78, + "y" : 442, + }, + ], +}, + +) diff --git a/test/results/rule34us.py b/test/results/rule34us.py new file mode 100644 index 00000000..43d8f66a --- /dev/null +++ b/test/results/rule34us.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import rule34us + + +__tests__ = ( +{ + "#url" : "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", + "#category": ("booru", "rule34us", "tag"), + "#class" : rule34us.Rule34usTagExtractor, + "#pattern" : r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", + "#count" : 10, +}, + +{ + "#url" : "https://rule34.us/index.php?r=posts/view&id=3709005", + "#category": ("booru", "rule34us", "post"), + "#class" : rule34us.Rule34usPostExtractor, + "#pattern" : r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", + "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", +}, + +{ + "#url" : "https://rule34.us/index.php?r=posts/view&id=4576310", + "#category": ("booru", "rule34us", "post"), + "#class" : rule34us.Rule34usPostExtractor, + "#pattern" : r"https://video\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", + + "extension" : "mp4", + "file_url" : str, + "filename" : "a294ff8e1f8e0efa041e5dc9d1480011", + "height" : "3982", + "id" : "4576310", + "md5" : "a294ff8e1f8e0efa041e5dc9d1480011", + "score" : r"re:\d+", + "tags" : "tagme, video", + "tags_general" : "video", + "tags_metadata": "tagme", + "uploader" : "Anonymous", + "width" : "3184", +}, + +) diff --git a/test/results/safebooru.py b/test/results/safebooru.py new file mode 100644 index 00000000..0c183204 --- /dev/null +++ b/test/results/safebooru.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url" : "17c61b386530cf4c30842c9f580d15ef1cd09586", + "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", +}, + +{ + "#url" : "https://safebooru.org/index.php?page=pool&s=show&id=11", + "#category": ("gelbooru_v02", "safebooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://safebooru.org/index.php?page=favorites&s=view&id=17567", + "#category": ("gelbooru_v02", "safebooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://safebooru.org/index.php?page=post&s=view&id=1169132", + "#category": ("gelbooru_v02", "safebooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : {"tags": True}, + "#sha1_url" : "cf05e37a3c62b2d55788e2080b8eabedb00f999b", + "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", + + "tags_artist" : "kawanakajima", + "tags_character": "heath_ledger ronald_mcdonald the_joker", + "tags_copyright": "dc_comics mcdonald's the_dark_knight", + "tags_general" : str, +}, + +) diff --git a/test/results/sakugabooru.py b/test/results/sakugabooru.py new file mode 100644 index 00000000..73d38316 --- /dev/null +++ b/test/results/sakugabooru.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://www.sakugabooru.com/post/show/125570", + "#category": ("moebooru", "sakugabooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/post?tags=nichijou", + "#category": ("moebooru", "sakugabooru", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/pool/show/54", + "#category": ("moebooru", "sakugabooru", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/post/popular_recent", + "#category": ("moebooru", "sakugabooru", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/sankaku.py b/test/results/sankaku.py new file mode 100644 index 00000000..e4fbf5c1 --- /dev/null +++ b/test/results/sankaku.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sankaku +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://sankaku.app/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", + "#count" : 5, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://black.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://white.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://sankaku.app/ja?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://sankaku.app/no/?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", + "#comment" : "error on five or more tags", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#options" : {"username": None}, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", + "#comment" : "match arbitrary query parameters", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=date:2023-03-20", + "#comment" : "'date:' tags (#1790)", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#range" : "1", + "#count" : 1, +}, + +{ + "#url" : "https://sankaku.app/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/pool/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, +}, + +{ + "#url" : "https://sankaku.app/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + + "tags_artist" : ["bonocho"], + "tags_studio" : ["dc_comics"], + "tags_medium" : list, + "tags_copyright": list, + "tags_character": list, + "tags_general" : list, +}, + +{ + "#url" : "https://sankaku.app/post/show/21418978", + "#comment" : "'contentious_content'", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", +}, + +{ + "#url" : "https://sankaku.app/post/show/20758561", + "#comment" : "empty tags (#1617)", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#options" : {"tags": True}, + "#count" : 1, + + "tags" : list, + "tags_general": [ + "key(mangaka)", + "key(mangaka)", + ], +}, + +{ + "#url" : "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", + "#comment" : "md5 hexdigest instead of ID (#3952)", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count" : 1, + + "id" : 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", +}, + +{ + "#url" : "https://chan.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/ja/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://white.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://black.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://sankaku.app/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class" : sankaku.SankakuBooksExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class" : sankaku.SankakuBooksExtractor, +}, + +) diff --git a/test/results/sankakucomplex.py b/test/results/sankakucomplex.py new file mode 100644 index 00000000..f944df16 --- /dev/null +++ b/test/results/sankakucomplex.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sankakucomplex + + +__tests__ = ( +{ + "#url" : "https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#sha1_url" : "4a9ecc5ae917fbce469280da5b6a482510cae84d", + "#sha1_metadata": "bfe08310e7d9a572f568f6900e0ed0eb295aa2b3", +}, + +{ + "#url" : "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#sha1_url" : "a1e249173fd6c899a8134fcfbd9c925588a63f7c", + "#sha1_metadata": "e78fcc23c2711befc0969a45ea5082a29efccf68", +}, + +{ + "#url" : "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", + "#comment" : "videos (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#pattern" : r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", + "#range" : "26-", + "#count" : 5, +}, + +{ + "#url" : "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", + "#comment" : "youtube embeds (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#options" : {"embeds": True}, + "#pattern" : "https://www.youtube.com/embed/", + "#range" : "2-", + "#count" : 2, +}, + +{ + "#url" : "https://www.sankakucomplex.com/tag/cosplay/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, + "#pattern" : sankakucomplex.SankakucomplexArticleExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.sankakucomplex.com/category/anime/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, +}, + +{ + "#url" : "https://www.sankakucomplex.com/author/rift/page/5/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, +}, + +) diff --git a/test/results/seiga.py b/test/results/seiga.py new file mode 100644 index 00000000..3749adc4 --- /dev/null +++ b/test/results/seiga.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import seiga +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, + "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", + "#count" : ">= 4", + + "user" : { + "id" : 39537793, + "message": str, + "name" : str, + }, + "clips" : int, + "comments" : int, + "count" : int, + "extension": None, + "image_id" : int, + "title" : str, + "views" : int, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/79433", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, +}, + +{ + "#url" : "https://sp.seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", + "#sha1_content" : "d9202292012178374d57fb0126f6124387265297", +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im123", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im10877923", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", +}, + +{ + "#url" : "https://seiga.nicovideo.jp/image/source/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/thumb/5977527i", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +) diff --git a/test/results/senmanga.py b/test/results/senmanga.py new file mode 100644 index 00000000..8192cfad --- /dev/null +++ b/test/results/senmanga.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import senmanga + + +__tests__ = ( +{ + "#url" : "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", + "#sha1_url" : "5f95140ff511d8497e2ec08fa7267c6bb231faec", + "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", + + "chapter" : "37A", + "count" : 2, + "extension": "", + "filename" : r"re:[12]", + "lang" : "ja", + "language" : "Japanese", + "manga" : "Bokura wa Minna Kawaisou", + "page" : int, +}, + +{ + "#url" : "http://raw.senmanga.com/Love-Lab/2016-03/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", + "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", + + "chapter" : "2016-03", + "count" : 9, + "extension": "", + "filename" : r"re:\d", + "manga" : "Renai Lab 恋愛ラボ", +}, + +{ + "#url" : "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", + + "chapter" : "1", + "count" : 65, + "extension": "jpg", + "filename" : r"re:\d+-\w+", + "manga" : "Akabane Honeko no Bodyguard", +}, + +{ + "#url" : "https://raw.senmanga.com/amama-cinderella/3", + "#comment" : "no http scheme ()", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", + "#count" : 30, +}, + +) diff --git a/test/results/sexcom.py b/test/results/sexcom.py new file mode 100644 index 00000000..079bb5fa --- /dev/null +++ b/test/results/sexcom.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sexcom + + +__tests__ = ( +{ + "#url" : "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", + "#comment" : "picture", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", + "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", + + "comments" : int, + "date" : "dt:2014-10-19 15:45:44", + "extension": "jpg", + "filename" : "7637609", + "likes" : int, + "pin_id" : 21241874, + "repins" : int, + "tags" : list, + "thumbnail": str, + "title" : "Sexy Ecchi Girls 166", + "type" : "picture", + "uploader" : "mangazeta", + "url" : str, +}, + +{ + "#url" : "https://www.sex.com/pin/55435122-ecchi/", + "#comment" : "gif", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", + "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", +}, + +{ + "#url" : "https://www.sex.com/pin/55748341/", + "#comment" : "video", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", + "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", +}, + +{ + "#url" : "https://www.sex.com/pin/55847384-very-nicely-animated/", + "#comment" : "pornhub embed", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", +}, + +{ + "#url" : "https://www.sex.com/pin/21241874/#related", + "#category": ("", "sexcom", "related-pin"), + "#class" : sexcom.SexcomRelatedPinExtractor, + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.sex.com/user/sirjuan79/pins/", + "#category": ("", "sexcom", "pins"), + "#class" : sexcom.SexcomPinsExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.sex.com/user/ronin17/exciting-hentai/", + "#category": ("", "sexcom", "board"), + "#class" : sexcom.SexcomBoardExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.sex.com/search/pics?query=ecchi", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.sex.com/videos/hentai/", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.sex.com/pics/?sort=popular&sub=all&page=1", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, +}, + +) diff --git a/test/results/simplyhentai.py b/test/results/simplyhentai.py new file mode 100644 index 00000000..561ce426 --- /dev/null +++ b/test/results/simplyhentai.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import simplyhentai +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, + "#sha1_url" : "21613585ae5ec2f69ea579e9713f536fceab5bd5", + "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", +}, + +{ + "#url" : "https://www.simply-hentai.com/notfound", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, + "#exception": exception.GalleryDLException, +}, + +{ + "#url" : "https://pokemon.simply-hentai.com/mao-friends-9bc39", + "#comment" : "custom subdomain", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, +}, + +{ + "#url" : "https://www.simply-hentai.com/vocaloid/black-magnet", + "#comment" : "www subdomain, two path segments", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, +}, + +{ + "#url" : "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", + "#category": ("", "simplyhentai", "image"), + "#class" : simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url" : "3d8eb55240a960134891bd77fe1df7988fcdc455", + "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", +}, + +{ + "#url" : "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", + "#category": ("", "simplyhentai", "image"), + "#class" : simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url" : "f73916527211b4a40f26568ee26cd8999f5f4f30", + "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", +}, + +{ + "#url" : "https://videos.simply-hentai.com/creamy-pie-episode-02", + "#category": ("", "simplyhentai", "video"), + "#class" : simplyhentai.SimplyhentaiVideoExtractor, + "#options" : {"verify": False}, + "#pattern" : r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", + "#count" : 1, + "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", +}, + +{ + "#url" : "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", + "#category": ("", "simplyhentai", "video"), + "#class" : simplyhentai.SimplyhentaiVideoExtractor, + "#sha1_url" : "ad9a36ae06c601b6490e3c401834b4949d947eb0", + "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", +}, + +) diff --git a/test/results/skeb.py b/test/results/skeb.py new file mode 100644 index 00000000..f956986b --- /dev/null +++ b/test/results/skeb.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import skeb + + +__tests__ = ( +{ + "#url" : "https://skeb.jp/@kanade_cocotte/works/38", + "#category": ("", "skeb", "post"), + "#class" : skeb.SkebPostExtractor, + "#count" : 2, + + "anonymous" : False, + "body" : r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", + "client" : { + "avatar_url" : r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", + "header_url" : r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", + "id" : 1196514, + "name" : str, + "screen_name": "minato_ragi", + }, + "content_category": "preview", + "creator" : { + "avatar_url" : "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", + "header_url" : "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", + "id" : 159273, + "name" : "イチノセ奏", + "screen_name": "kanade_cocotte", + }, + "file_id" : int, + "file_url" : str, + "genre" : "art", + "nsfw" : False, + "original" : { + "byte_size" : int, + "duration" : None, + "extension" : r"re:psd|png", + "frame_rate": None, + "height" : 3727, + "is_movie" : False, + "width" : 2810, + }, + "post_num" : "38", + "post_url" : "https://skeb.jp/@kanade_cocotte/works/38", + "source_body" : None, + "source_thanks" : None, + "tags" : list, + "thanks" : None, + "translated_body" : False, + "translated_thanks": None, +}, + +{ + "#url" : "https://skeb.jp/@kanade_cocotte", + "#category": ("", "skeb", "user"), + "#class" : skeb.SkebUserExtractor, + "#pattern" : r"https://skeb\.imgix\.net/uploads/origins/[\w-]+\?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&fm=webp&w=800&s=\w+", + "#range" : "1-5", +}, + +{ + "#url" : "https://skeb.jp/search?q=bunny%20tree&t=works", + "#category": ("", "skeb", "search"), + "#class" : skeb.SkebSearchExtractor, + "#count" : ">= 18", + + "search_tags": "bunny tree", +}, + +{ + "#url" : "https://skeb.jp/@user/following_creators", + "#category": ("", "skeb", "following"), + "#class" : skeb.SkebFollowingExtractor, +}, + +) diff --git a/test/results/slickpic.py b/test/results/slickpic.py new file mode 100644 index 00000000..ff1eb1ff --- /dev/null +++ b/test/results/slickpic.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import slickpic + + +__tests__ = ( +{ + "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class" : slickpic.SlickpicAlbumExtractor, + "#pattern" : r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", + "#count" : 102, + "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class" : slickpic.SlickpicAlbumExtractor, + "#range" : "34", + "#sha1_content": [ + "276eb2c902187bb177ae8013e310e1d6641fba9a", + "52b5a310587de1048030ab13a912f6a3a9cc7dab", + "cec6630e659dc72db1ee1a9a6f3b525189261988", + "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", + "22e83645fc242bc3584eca7ec982c8a53a4d8a44", + ], +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/gallery/", + "#category": ("", "slickpic", "user"), + "#class" : slickpic.SlickpicUserExtractor, + "#pattern" : slickpic.SlickpicAlbumExtractor.pattern, + "#count" : ">= 358", +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/", + "#category": ("", "slickpic", "user"), + "#class" : slickpic.SlickpicUserExtractor, +}, + +) diff --git a/test/results/slideshare.py b/test/results/slideshare.py new file mode 100644 index 00000000..561f0676 --- /dev/null +++ b/test/results/slideshare.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import slideshare + + +__tests__ = ( +{ + "#url" : "https://www.slideshare.net/Slideshare/get-started-with-slide-share", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#pattern" : r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/get-started-with-slide-share-\d+-1024\.jpg\?cb=\d+", + "#count" : 19, + "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", + + "description" : "Get Started with SlideShare - A Beginngers Guide for Creators", + "likes" : int, + "presentation": "get-started-with-slide-share", + "date" : "dt:2015-05-20 17:38:21", + "title" : "Getting Started With SlideShare", + "user" : "Slideshare", + "views" : int, +}, + +{ + "#url" : "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", + "#comment" : "long title and description", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#sha1_url": "d8952260f8bec337dd809a958ec8091350393f6b", + + "title" : "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", + "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", +}, + +{ + "#url" : "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", + "#comment" : "mobile URL", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#sha1_url": "72c431cb1eccbb6794f608ecbbc01d52e8768159", +}, + +) diff --git a/test/results/smugloli.py b/test/results/smugloli.py new file mode 100644 index 00000000..5d044123 --- /dev/null +++ b/test/results/smugloli.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://smuglo.li/a/res/1154380.html", + "#category": ("vichan", "smugloli", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://smug.+/a/src/\d+(-\d)?\.\w+", + "#count" : ">= 18", + + "board" : "a", + "thread": "1154380", + "title" : "Mob Psycho 100 Season 3", +}, + +{ + "#url" : "https://smugloli.net/a/res/1145409.html", + "#category": ("vichan", "smugloli", "thread"), + "#class" : vichan.VichanThreadExtractor, +}, + +{ + "#url" : "https://smuglo.li/a", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://smuglo.li/a/1.html", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://smugloli.net/cute/catalog.html", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/smugmug.py b/test/results/smugmug.py new file mode 100644 index 00000000..fd30a6b9 --- /dev/null +++ b/test/results/smugmug.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import smugmug + + +__tests__ = ( +{ + "#url" : "smugmug:album:cr4C7f", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#sha1_url": "2c2e576e47d4e9ce60b44871f08a8c66b5ebaceb", +}, + +{ + "#url" : "smugmug:album:Fb7hMs", + "#comment" : "empty", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "smugmug:album:6VRT8G", + "#comment" : "no 'User'", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#sha1_url": "c4a0f4c4bfd514b93cbdeb02b3345bf7ef6604df", +}, + +{ + "#url" : "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", + "#category": ("", "smugmug", "image"), + "#class" : smugmug.SmugmugImageExtractor, + "#sha1_url" : "e6408fd2c64e721fd146130dceb56a971ceb4259", + "#sha1_metadata": "b31a63d07c9c26eb0f79f52d60d171a98938f99b", + "#sha1_content" : "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", +}, + +{ + "#url" : "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", + "#comment" : "video", + "#category": ("", "smugmug", "image"), + "#class" : smugmug.SmugmugImageExtractor, + "#sha1_url" : "04d0ab1ff829ca7d78f5acb5548953df08e9a5ee", + "#sha1_metadata": "2b545184592c282b365fcbb7df6ca7952b8a3173", +}, + +{ + "#url" : "https://tdm.smugmug.com/Nature/Dove", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:cr4C7f$", +}, + +{ + "#url" : "https://tdm.smugmug.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : smugmug.SmugmugAlbumExtractor.pattern, + "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", +}, + +{ + "#url" : "https://www.smugmug.com/gallery/n-GLCjnD/", + "#comment" : "gallery node without owner", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:6VRT8G$", +}, + +{ + "#url" : "smugmug:www.sitkapics.com/TREES-and-TRAILS/", + "#comment" : "custom domain", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:ct8Nds$", +}, + +{ + "#url" : "smugmug:www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : r"smugmug:album:\w{6}$", + "#count" : ">= 14", +}, + +{ + "#url" : "smugmug:https://www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, +}, + +) diff --git a/test/results/soundgasm.py b/test/results/soundgasm.py new file mode 100644 index 00000000..16f0a17c --- /dev/null +++ b/test/results/soundgasm.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import soundgasm + + +__tests__ = ( +{ + "#url" : "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class" : soundgasm.SoundgasmAudioExtractor, + "#pattern" : r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", + + "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", + "extension" : "m4a", + "filename" : "26cb2b23b2f2c6094b40ee3a9167271e274b570a", + "slug" : "687-Otto-von-Toontown-12822", + "title" : "687 - Otto von Toontown (12/8/22)", + "user" : "ClassWarAndPuppies2", +}, + +{ + "#url" : "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class" : soundgasm.SoundgasmAudioExtractor, +}, + +{ + "#url" : "https://soundgasm.net/u/fierce-aphrodite", + "#category": ("", "soundgasm", "user"), + "#class" : soundgasm.SoundgasmUserExtractor, + "#pattern" : r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", + "#count" : ">= 15", + + "description": str, + "extension" : "m4a", + "filename" : r"re:^[0-9a-f]{40}$", + "slug" : str, + "title" : str, + "url" : str, + "user" : "fierce-aphrodite", +}, + +) diff --git a/test/results/speakerdeck.py b/test/results/speakerdeck.py new file mode 100644 index 00000000..5e63a4cb --- /dev/null +++ b/test/results/speakerdeck.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import speakerdeck + + +__tests__ = ( +{ + "#url" : "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", + "#category": ("", "speakerdeck", "presentation"), + "#class" : speakerdeck.SpeakerdeckPresentationExtractor, + "#pattern" : r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", + "#count" : 6, + "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", +}, + +) diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py new file mode 100644 index 00000000..8fe6d8dc --- /dev/null +++ b/test/results/subscribestar.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import subscribestar +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#pattern" : r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", + "#count" : ">= 20", + + "author_id" : 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content" : str, + "date" : datetime.datetime, + "id" : int, + "num" : int, + "post_id" : int, + "type" : r"re:image|video|attachment", + "url" : str, + "?pinned" : bool, +}, + +{ + "#url" : "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#options" : {"metadata": True}, + "#range" : "1", + + "date": datetime.datetime, +}, + +{ + "#url" : "https://subscribestar.adult/kanashiipanda", + "#category": ("", "subscribestar", "user-adult"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.subscribestar.com/posts/102468", + "#category": ("", "subscribestar", "post"), + "#class" : subscribestar.SubscribestarPostExtractor, + "#count" : 1, + + "author_id" : 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content" : r"re:<h1>Brand Guidelines and Assets</h1>", + "date" : "dt:2020-05-07 12:33:00", + "extension" : "jpg", + "filename" : "8ff61299-b249-47dc-880a-cdacc9081c62", + "group" : "imgs_and_videos", + "height" : 291, + "id" : 203885, + "num" : 1, + "pinned" : False, + "post_id" : 102468, + "type" : "image", + "width" : 700, +}, + +{ + "#url" : "https://subscribestar.adult/posts/22950", + "#category": ("", "subscribestar", "post-adult"), + "#class" : subscribestar.SubscribestarPostExtractor, + "#count" : 1, + + "date": "dt:2019-04-28 07:32:00", +}, + +) diff --git a/test/results/sushiski.py b/test/results/sushiski.py new file mode 100644 index 00000000..e1bbd20a --- /dev/null +++ b/test/results/sushiski.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://sushi.ski/@ui@misskey.04.si", + "#category": ("misskey", "sushi.ski", "user"), + "#class" : misskey.MisskeyUserExtractor, +}, + +{ + "#url" : "https://sushi.ski/@hatusimo_sigure/following", + "#category": ("misskey", "sushi.ski", "following"), + "#class" : misskey.MisskeyFollowingExtractor, +}, + +{ + "#url" : "https://sushi.ski/notes/9bm3x4ksqw", + "#category": ("misskey", "sushi.ski", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#pattern" : r"https://media\.sushi\.ski/files/[\w-]+\.png", + "#count" : 1, +}, + +{ + "#url" : "https://sushi.ski/my/favorites", + "#category": ("misskey", "sushi.ski", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/tapas.py b/test/results/tapas.py new file mode 100644 index 00000000..a3d79842 --- /dev/null +++ b/test/results/tapas.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tapas + + +__tests__ = ( +{ + "#url" : "https://tapas.io/series/just-leave-me-be", + "#category": ("", "tapas", "series"), + "#class" : tapas.TapasSeriesExtractor, + "#pattern" : r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", + "#count" : 132, +}, + +{ + "#url" : "https://tapas.io/series/yona", + "#comment" : "mature", + "#category": ("", "tapas", "series"), + "#class" : tapas.TapasSeriesExtractor, + "#count" : 26, +}, + +{ + "#url" : "https://tapas.io/episode/2068651", + "#category": ("", "tapas", "episode"), + "#class" : tapas.TapasEpisodeExtractor, + "#pattern" : "^text:", + "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", + + "book" : True, + "comment_cnt" : int, + "date" : "dt:2021-02-23 16:02:07", + "early_access" : False, + "escape_title" : "You are a Tomb Raider (2)", + "free" : True, + "id" : 2068651, + "like_cnt" : int, + "liked" : bool, + "mature" : False, + "next_ep_id" : 2068652, + "nsfw" : False, + "nu" : False, + "num" : 1, + "open_comments" : True, + "pending_scene" : 2, + "prev_ep_id" : 2068650, + "publish_date" : "2021-02-23T16:02:07Z", + "read" : bool, + "related_ep_id" : None, + "relative_publish_date": "Feb 23, 2021", + "scene" : 2, + "scheduled" : False, + "title" : "You are a Tomb Raider (2)", + "unlock_cnt" : 0, + "unlocked" : False, + "view_cnt" : int, + "series" : { + "genre" : dict, + "has_book_cover": True, + "has_top_banner": True, + "id" : 199931, + "premium" : True, + "sale_type" : "PAID", + "subscribed" : bool, + "thumbsup_cnt" : int, + "title" : "Tomb Raider King", + "type" : "BOOKS", + "url" : "tomb-raider-king-novel", + }, +}, + +) diff --git a/test/results/tbib.py b/test/results/tbib.py new file mode 100644 index 00000000..e4481da5 --- /dev/null +++ b/test/results/tbib.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", + "#category": ("gelbooru_v02", "tbib", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : ">= 120", +}, + +{ + "#url" : "https://tbib.org/index.php?page=favorites&s=view&id=7881", + "#category": ("gelbooru_v02", "tbib", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://tbib.org/index.php?page=post&s=view&id=9233957", + "#category": ("gelbooru_v02", "tbib", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_url" : "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", + "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", +}, + +) diff --git a/test/results/tcbscans.py b/test/results/tcbscans.py new file mode 100644 index 00000000..1b77a79b --- /dev/null +++ b/test/results/tcbscans.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tcbscans + + +__tests__ = ( +{ + "#url" : "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, + "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count" : 17, + + "manga" : "Chainsaw Man", + "chapter" : 108, + "chapter_minor": "", + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, + "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count" : 18, + + "manga" : "One Piece", + "chapter" : 1065, + "chapter_minor": "", + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, +}, + +{ + "#url" : "https://tcbscans.com/mangas/13/chainsaw-man", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, + "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, + "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://onepiecechapters.com/mangas/15/hunter-x-hunter", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, +}, + +) diff --git a/test/results/tco.py b/test/results/tco.py new file mode 100644 index 00000000..37a1b106 --- /dev/null +++ b/test/results/tco.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import urlshortener +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://t.co/bCgBY8Iv5n", + "#category": ("urlshortener", "tco", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#pattern" : "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", + "#count" : 1, +}, + +{ + "#url" : "https://t.co/abcdefghij", + "#category": ("urlshortener", "tco", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/telegraph.py b/test/results/telegraph.py new file mode 100644 index 00000000..726981b7 --- /dev/null +++ b/test/results/telegraph.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import telegraph + + +__tests__ = ( +{ + "#url" : "https://telegra.ph/Telegraph-Test-03-28", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.png", + + "author" : "mikf", + "caption" : r"re:test|", + "count" : 2, + "date" : "dt:2022-03-28 16:01:36", + "description": "Just a test", + "post_url" : "https://telegra.ph/Telegraph-Test-03-28", + "slug" : "Telegraph-Test-03-28", + "title" : "Telegra.ph Test", +}, + +{ + "#url" : "https://telegra.ph/森-03-28", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + "#count" : 1, + + "author" : "&", + "caption" : "kokiri", + "count" : 1, + "date" : "dt:2022-03-28 16:31:26", + "description" : "コキリの森", + "extension" : "jpg", + "filename" : "3ea79d23b0dd0889f215a", + "num" : 1, + "num_formatted": "1", + "post_url" : "https://telegra.ph/森-03-28", + "slug" : "森-03-28", + "title" : "\"森\"", + "url" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", +}, + +{ + "#url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", + "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", + + "author" : "Shotacon - заходи сюда", + "caption" : "", + "count" : 19, + "date" : "dt:2022-05-27 16:17:27", + "description" : "", + "num_formatted": r"re:^\d{2}$", + "post_url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "slug" : "Vsyo-o-druzyah-moej-sestricy-05-27", + "title" : "Всё о друзьях моей сестрицы", +}, + +{ + "#url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", + + "author" : "cosmos", + "caption" : "", + "count" : 89, + "date" : "dt:2022-02-21 05:57:39", + "description" : "", + "num_formatted": r"re:^\d{2}$", + "post_url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "slug" : "Disharmonica---Saber-Nero-02-21", + "title" : "Disharmonica - Saber Nero", +}, + +) diff --git a/test/results/tentaclerape.py b/test/results/tentaclerape.py new file mode 100644 index 00000000..247067f6 --- /dev/null +++ b/test/results/tentaclerape.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://tentaclerape.net/post/list/comic/1", + "#category": ("shimmie2", "tentaclerape", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://tentaclerape.net/post/view/10", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", + "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", + + "extension" : "jpg", + "file_url" : "https://tentaclerape.net/./index.php?q=/image/10.jpg", + "filename" : "10", + "height" : 427, + "id" : 10, + "md5" : "945db71eeccaef82ce44b77564260c0b", + "size" : 0, + "subcategory": "post", + "tags" : "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", + "width" : 300, +}, + +{ + "#url" : "https://tentaclerape.net/post/view/91267", + "#comment" : "video", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", +}, + +) diff --git a/test/results/test.py b/test/results/test.py new file mode 100644 index 00000000..54566cc7 --- /dev/null +++ b/test/results/test.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import test + + +__tests__ = ( +{ + "#url" : "test:pixiv", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +{ + "#url" : "test:pixiv:user,favorite:0", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +{ + "#url" : "test:", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +) diff --git a/test/results/thebarchive.py b/test/results/thebarchive.py new file mode 100644 index 00000000..5fd6e226 --- /dev/null +++ b/test/results/thebarchive.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://thebarchive.com/b/thread/739772332/", + "#category": ("foolfuuka", "thebarchive", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", +}, + +{ + "#url" : "https://thebarchive.com/b/", + "#category": ("foolfuuka", "thebarchive", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://thebarchive.com/_/search/text/test/", + "#category": ("foolfuuka", "thebarchive", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://thebarchive.com/b/gallery/9", + "#category": ("foolfuuka", "thebarchive", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/thecollection.py b/test/results/thecollection.py new file mode 100644 index 00000000..abe47380 --- /dev/null +++ b/test/results/thecollection.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", + "#category": ("gelbooru_v01", "thecollection", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", + "#category": ("gelbooru_v01", "thecollection", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", + "#category": ("gelbooru_v01", "thecollection", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", + "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", +}, + +) diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py new file mode 100644 index 00000000..22bc1412 --- /dev/null +++ b/test/results/toyhouse.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import toyhouse +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.toyhou.se/d-floe/art", + "#category": ("", "toyhouse", "art"), + "#class" : toyhouse.ToyhouseArtExtractor, + "#pattern" : r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", + "#range" : "1-30", + "#count" : 30, + + "artists" : list, + "characters": list, + "date" : datetime.datetime, + "hash" : r"re:\w+", + "id" : r"re:\d+", + "url" : str, + "user" : "d-floe", +}, + +{ + "#url" : "https://www.toyhou.se/kroksoc/art", + "#comment" : "protected by Content Warning", + "#category": ("", "toyhouse", "art"), + "#class" : toyhouse.ToyhouseArtExtractor, + "#count" : ">= 19", +}, + +{ + "#url" : "https://toyhou.se/~images/40587320", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, + "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", + + "artists" : ["d-floe"], + "characters": ["Sumi"], + "date" : "dt:2021-10-08 01:32:47", + "extension" : "png", + "filename" : "40587320_TT1NaBUr3FLkS1p", + "hash" : "TT1NaBUr3FLkS1p", + "id" : "40587320", + "url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", +}, + +{ + "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", + "#comment" : "direct link, multiple artists", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, + + "artists" : [ + "http://aminoapps.com/p/92sf3z", + "kroksoc (Color)", + ], + "characters": ["❀Reiichi❀"], + "date" : "dt:2021-07-03 20:02:02", + "hash" : "bqhGcwcnU", + "id" : "36817425", +}, + +{ + "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, +}, + +) diff --git a/test/results/tsumino.py b/test/results/tsumino.py new file mode 100644 index 00000000..cbe2c77a --- /dev/null +++ b/test/results/tsumino.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tsumino + + +__tests__ = ( +{ + "#url" : "https://www.tsumino.com/entry/40996", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, + "#pattern" : r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", + + "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", + "gallery_id": 40996, + "date" : "dt:2018-06-29 00:00:00", + "count" : 42, + "collection": "", + "artist" : ["Itou Life"], + "group" : ["Itou Life"], + "parody" : list, + "characters": list, + "tags" : list, + "type" : "Doujinshi", + "rating" : float, + "uploader" : "sehki", + "lang" : "en", + "language" : "English", + "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", +}, + +{ + "#url" : "https://www.tsumino.com/Book/Info/40996", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Read/View/45834", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Read/Index/45834", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", + "#category": ("", "tsumino", "search"), + "#class" : tsumino.TsuminoSearchExtractor, + "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", + "#category": ("", "tsumino", "search"), + "#class" : tsumino.TsuminoSearchExtractor, + "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, + "#count" : ">= 3", +}, + +) diff --git a/test/results/tumblr.py b/test/results/tumblr.py new file mode 100644 index 00000000..99883578 --- /dev/null +++ b/test/results/tumblr.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tumblr +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : {"posts": "photo"}, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", + "#count" : 1, +}, + +{ + "#url" : "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "posts" : "all", + "external": True, + }, + "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+_1280\.jpg|a\.tumblr\.com/tumblr_\w+)", + "#count" : 3, +}, + +{ + "#url" : "https://mikf123-hidden.tumblr.com/", + "#comment" : "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : {"access-token": None}, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://mikf123-hidden.tumblr.com/", + "#comment" : "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "hidden", + ], +}, + +{ + "#url" : "https://mikf123-private.tumblr.com/", + "#comment" : "password protected", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "private", + ], +}, + +{ + "#url" : "https://mikf123-private-hidden.tumblr.com/", + "#comment" : "dashboard-only & password protected", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "private", + "hidden", + ], +}, + +{ + "#url" : "https://mikf123.tumblr.com/", + "#comment" : "date-min/-max/-format (#337)", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "date-min" : "201804", + "date-max" : "201805", + "date-format": "%Y%m", + }, + "#count" : 4, +}, + +{ + "#url" : "https://donttrustthetits.tumblr.com/", + "#comment" : "pagination with 'date-max' (#2191) and 'api-key'", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "access-token": None, + "original" : False, + "date-max" : "2015-04-25T00:00:00", + "date-min" : "2015-04-01T00:00:00", + }, + "#count" : 316, +}, + +{ + "#url" : "https://demo.tumblr.com/page/2", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://demo.tumblr.com/archive", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "tumblr:http://www.b-authentique.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "tumblr:www.b-authentique.com", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "http://demo.tumblr.com/post/459265350", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167770226574/text-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/181022561719/quote-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167623351559/link-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167633596145/video-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167770026604/audio-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/172687798174/photo-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/181022380064/chat-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://kichatundk.tumblr.com/post/654953419288821760", + "#comment" : "high-quality images (#1846)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, + "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", +}, + +{ + "#url" : "https://hameru-is-cool.tumblr.com/post/639261855227002880", + "#comment" : "high-quality images (#1344)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, + "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", +}, + +{ + "#url" : "https://mikf123.tumblr.com/image/689860196535762944", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#pattern" : r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", +}, + +{ + "#url" : "http://ziemniax.tumblr.com/post/109697912859/", + "#comment" : "HTML response (#297)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "http://demo.tumblr.com/image/459265350", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "http://demo.tumblr.com/tagged/Times%20Square", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://mikf123.tumblr.com/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, + "#pattern" : r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", + "#count" : 1, + + "id": 169341068404, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +) diff --git a/test/results/tumblrgallery.py b/test/results/tumblrgallery.py new file mode 100644 index 00000000..a1a6eeb5 --- /dev/null +++ b/test/results/tumblrgallery.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tumblrgallery + + +__tests__ = ( +{ + "#url" : "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", + "#category": ("", "tumblrgallery", "tumblrblog"), + "#class" : tumblrgallery.TumblrgalleryTumblrblogExtractor, +}, + +{ + "#url" : "https://tumblrgallery.xyz/post/405674.html", + "#category": ("", "tumblrgallery", "post"), + "#class" : tumblrgallery.TumblrgalleryPostExtractor, + "#pattern" : r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", + "#count" : 3, +}, + +{ + "#url" : "https://tumblrgallery.xyz/s.php?q=everyday-life", + "#category": ("", "tumblrgallery", "search"), + "#class" : tumblrgallery.TumblrgallerySearchExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/.+", + "#count" : "< 1000", +}, + +) diff --git a/test/results/turboimagehost.py b/test/results/turboimagehost.py new file mode 100644 index 00000000..3e2069fa --- /dev/null +++ b/test/results/turboimagehost.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://www.turboimagehost.com/p/39078423/test--.png.html", + "#category": ("imagehost", "turboimagehost", "image"), + "#class" : imagehosts.TurboimagehostImageExtractor, + "#sha1_url" : "b94de43612318771ced924cb5085976f13b3b90e", + "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", + "#sha1_content" : "f38b54b17cd7462e687b58d83f00fca88b1b105a", +}, + +) diff --git a/test/results/twibooru.py b/test/results/twibooru.py new file mode 100644 index 00000000..5dd0191d --- /dev/null +++ b/test/results/twibooru.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import twibooru + + +__tests__ = ( +{ + "#url" : "https://twibooru.org/1", + "#category": ("philomena", "twibooru", "post"), + "#class" : twibooru.TwibooruPostExtractor, + "#pattern" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", + + "animated" : False, + "aspect_ratio" : 1.0, + "comment_count" : int, + "created_at" : "2020-07-08T22:26:55.743Z", + "date" : "dt:2020-07-08 22:26:55", + "description" : "Why have I done this?", + "downvotes" : 0, + "duration" : 0.0, + "faves" : int, + "first_seen_at" : "2020-07-08T22:26:55.743Z", + "format" : "png", + "height" : 576, + "hidden_from_users": False, + "id" : 1, + "intensities" : dict, + "locations" : [], + "media_type" : "image", + "mime_type" : "image/png", + "name" : "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", + "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", + "processed" : True, + "representations" : dict, + "score" : int, + "sha512_hash" : "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", + "size" : 70910, + "source_url" : "", + "tag_ids" : list, + "tags" : list, + "thumbnails_generated": True, + "updated_at" : "2022-11-27T00:34:50.483Z", + "upvotes" : int, + "view_url" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "width" : 576, + "wilson_score" : float, +}, + +{ + "#url" : "https://twibooru.org/search?q=cute", + "#category": ("philomena", "twibooru", "search"), + "#class" : twibooru.TwibooruSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://twibooru.org/tags/cute", + "#category": ("philomena", "twibooru", "search"), + "#class" : twibooru.TwibooruSearchExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://twibooru.org/galleries/1", + "#category": ("philomena", "twibooru", "gallery"), + "#class" : twibooru.TwibooruGalleryExtractor, + "#range" : "1-20", + + "gallery": { + "description" : "Best nation pone and russian related pics.", + "id" : 1, + "spoiler_warning": "Russia", + "thumbnail_id" : 694923, + "title" : "Marussiaverse", + }, +}, + +) diff --git a/test/results/twitter.py b/test/results/twitter.py new file mode 100644 index 00000000..4094bc94 --- /dev/null +++ b/test/results/twitter.py @@ -0,0 +1,574 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import twitter +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://twitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : r"https://twitter\.com/supernaturepics/(photo|header_photo|timeline|tweets|media|with_replies|likes)$", + "#count" : 7, +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics?p=i", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/i/user/2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/intent/user?user_id=2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://fxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://vxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://x.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/timeline", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://twitter.com/OptionalTypo/timeline", + "#comment" : "suspended account (#2216)", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://twitter.com/id:772949683521978368/timeline", + "#comment" : "suspended account user ID", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/timeline#t", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/timeline", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/tweets", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/tweets#t", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/tweets", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/with_replies", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/with_replies#t", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/with_replies", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/media", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/media#t", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/media", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/likes", + "#category": ("", "twitter", "likes"), + "#class" : twitter.TwitterLikesExtractor, +}, + +{ + "#url" : "https://twitter.com/i/bookmarks", + "#category": ("", "twitter", "bookmark"), + "#class" : twitter.TwitterBookmarkExtractor, +}, + +{ + "#url" : "https://twitter.com/i/lists/784214683683127296", + "#category": ("", "twitter", "list"), + "#class" : twitter.TwitterListExtractor, + "#range" : "1-40", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://twitter.com/i/lists/784214683683127296/members", + "#category": ("", "twitter", "list-members"), + "#class" : twitter.TwitterListMembersExtractor, + "#pattern" : twitter.TwitterTimelineExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/following", + "#category": ("", "twitter", "following"), + "#class" : twitter.TwitterFollowingExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/following", + "#category": ("", "twitter", "following"), + "#class" : twitter.TwitterFollowingExtractor, +}, + +{ + "#url" : "https://twitter.com/search?q=nature", + "#category": ("", "twitter", "search"), + "#class" : twitter.TwitterSearchExtractor, + "#range" : "1-20", + "#count" : 20, + "#archive" : False, +}, + +{ + "#url" : "https://twitter.com/hashtag/nature", + "#category": ("", "twitter", "hashtag"), + "#class" : twitter.TwitterHashtagExtractor, + "#pattern" : twitter.TwitterSearchExtractor.pattern, + "#sha1_url": "3571c3a53b7647ea35517041fdc17f77ec5b2cb9", +}, + +{ + "#url" : "https://twitter.com/i/events/1484669206993903616", + "#category": ("", "twitter", "event"), + "#class" : twitter.TwitterEventExtractor, + "#range" : "1-20", + "#count" : ">=1", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment" : "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#sha1_url" : "88a40f7d25529c2501c46f2218f9e0de9aa634b4", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", +}, + +{ + "#url" : "https://twitter.com/perrypumas/status/894001459754180609", + "#comment" : "4 images", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", +}, + +{ + "#url" : "https://twitter.com/perrypumas/status/1065692031626829824", + "#comment" : "video", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", +}, + +{ + "#url" : "https://twitter.com/playpokemon/status/1263832915173048321", + "#comment" : "content with emoji, newlines, hashtags (#338)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! + +You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! + +❓🎁➡️ """, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1170041925560258560", + "#comment" : "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1170041925560258560", + "#comment" : "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": False}, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1424882930803908612", + "#comment" : "'replies' to self (#1254)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": "self"}, + "#count" : 4, + + "user": { + "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", + "url" : "http://princecanary.tumblr.com", + }, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1424898916156284928", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": "self"}, + "#count" : 1, +}, + +{ + "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"quoted": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", + "#count" : 8, +}, + +{ + "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : r"https://pbs\.twimg\.com/media/EaK.+=jpg", + "#count" : 4, +}, + +{ + "#url" : "https://twitter.com/web/status/1644907989109751810", + "#comment" : "different 'user' and 'author' in quoted Tweet (#3922)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "author": { + "id" : 321629993, + "name": "Cakes_Comics", + }, + "user" : { + "id" : 718928225360080897, + "name": "StobiesGalaxy", + }, +}, + +{ + "#url" : "https://twitter.com/i/web/status/112900228289540096", + "#comment" : "TwitPic embeds (#579)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : { + "twitpic": True, + "cards" : False, + }, + "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.jpg", + "#count" : 2, +}, + +{ + "#url" : "https://twitter.com/shimoigusaP/status/8138669971", + "#comment" : "TwitPic URL not in 'urls' (#3792)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"twitpic": True}, + "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.png", + "#count" : 1, +}, + +{ + "#url" : "https://twitter.com/billboard/status/1306599586602135555", + "#comment" : "Twitter card (#1005)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs.twimg.com/card_img/\d+/", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1561674543323910144", + "#comment" : "unified_card image_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/F.+=jpg", +}, + +{ + "#url" : "https://twitter.com/doax_vv_staff/status/1479438945662685184", + "#comment" : "unified_card image_carousel_website", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/F.+=png", + "#count" : 6, +}, + +{ + "#url" : "https://twitter.com/bang_dream_1242/status/1561548715348746241", + "#comment" : "unified_card video_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1466183847628865544", + "#comment" : "unified_card without type", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1571141912295243776", + "#comment" : "'cards-blacklist' option", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : { + "cards" : "ytdl", + "cards-blacklist": ("twitch.tv",), + }, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment" : "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": True}, + "#count" : 2, + + "tweet_id" : 1296304589591810048, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:34:32", + "date_original": "dt:2020-08-20 04:00:28", +}, + +{ + "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment" : "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": "original"}, + "#count" : 2, + + "tweet_id" : 1296296016002547713, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:00:28", + "date_original": "dt:2020-08-20 04:00:28", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment" : "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"conversations": True}, + "#count" : 5, +}, + +{ + "#url" : "https://twitter.com/morino_ya/status/1392763691599237121", + "#comment" : "retweet with missing media entities (#1555)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": True}, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1460044411165888515", + "#comment" : "deleted quote tweet (#2225)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1486373748911575046", + "#comment" : "'Misleading' content", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://twitter.com/mightbecursed/status/1492954264909479936", + "#comment" : "age-restricted (#2354)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"syndication": True}, + "#count" : 1, + + "date": "dt:2022-02-13 20:10:09", +}, + +{ + "#url" : "https://twitter.com/my0nruri/status/1528379296041299968", + "#comment" : "media alt texts / descriptions (#2617)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "description": "oc", +}, + +{ + "#url" : "https://twitter.com/poco_dandy/status/1150646424461176832", + "#comment" : "'?format=...&name=...'-style URLs", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs.twimg.com/card_img/157\d+/[\w-]+\?format=(jpg|png)&name=orig$", + "#range" : "1-2", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1629193457112686592", + "#comment" : "note tweet with long 'content'", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. + +The Washington Post writes, "Three weeks after the toxic train derailment in Ohio, an analysis of Environmental Protection Agency data has found nine air pollutants at levels that could raise long-term health concerns in and around East Palestine, according to an independent analysis. + +"The analysis by Texas A&M University seems to contradict statements by state and federal regulators that air near the crash site is completely safe, despite residents complaining about rashes, breathing problems and other health effects." Your reaction.""", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/photo", + "#category": ("", "twitter", "avatar"), + "#class" : twitter.TwitterAvatarExtractor, + "#pattern" : r"https://pbs\.twimg\.com/profile_images/554585280938659841/FLVAlX18\.jpeg", + + "date" : "dt:2015-01-12 10:26:49", + "extension": "jpeg", + "filename" : "FLVAlX18", + "tweet_id" : 554585280938659841, +}, + +{ + "#url" : "https://twitter.com/User16/photo", + "#category": ("", "twitter", "avatar"), + "#class" : twitter.TwitterAvatarExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/header_photo", + "#category": ("", "twitter", "background"), + "#class" : twitter.TwitterBackgroundExtractor, + "#pattern" : r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", + + "date" : "dt:2015-01-12 10:29:43", + "filename": "1421058583", + "tweet_id": 554586009367478272, +}, + +{ + "#url" : "https://twitter.com/User16/header_photo", + "#category": ("", "twitter", "background"), + "#class" : twitter.TwitterBackgroundExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", + "#category": ("", "twitter", "image"), + "#class" : twitter.TwitterImageExtractor, + "#options" : {"size": "4096x4096,orig"}, + "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", +}, + +{ + "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", + "#category": ("", "twitter", "image"), + "#class" : twitter.TwitterImageExtractor, +}, + +) diff --git a/test/results/unique-vintage.py b/test/results/unique-vintage.py new file mode 100644 index 00000000..8fe37a3b --- /dev/null +++ b/test/results/unique-vintage.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.unique-vintage.com/collections/flapper-1920s", + "#category": ("shopify", "unique-vintage", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", + "#category": ("shopify", "unique-vintage", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/unsplash.py b/test/results/unsplash.py new file mode 100644 index 00000000..1568ed3a --- /dev/null +++ b/test/results/unsplash.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import unsplash + + +__tests__ = ( +{ + "#url" : "https://unsplash.com/photos/lsoogGC_5dg", + "#category": ("", "unsplash", "image"), + "#class" : unsplash.UnsplashImageExtractor, + "#pattern" : r"https://images\.unsplash\.com/photo-1586348943529-beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", + + "alt_description": r"re:silhouette of trees near body of water ", + "blur_hash" : "LZP4uQS4jboe%#o0WCa}2doJNaaz", + "? categories" : list, + "color" : "#f3c08c", + "created_at" : "2020-04-08T12:29:42Z", + "date" : "dt:2020-04-08 12:29:42", + "description" : "The Island", + "downloads" : int, + "exif" : { + "aperture" : "11", + "exposure_time": "30", + "focal_length" : "70.0", + "iso" : 200, + "make" : "Canon", + "model" : "Canon EOS 5D Mark IV", + }, + "extension" : "jpg", + "filename" : "photo-1586348943529-beaae6c28db9", + "height" : 6272, + "id" : "lsoogGC_5dg", + "liked_by_user" : False, + "likes" : int, + "location" : { + "city" : "Beaver Dam", + "country" : "United States", + "name" : "Beaver Dam, WI 53916, USA", + "position": { + "latitude" : 43.457769, + "longitude": -88.837329, + }, + }, + "promoted_at" : "2020-04-08T15:12:03Z", + "sponsorship" : None, + "tags" : list, + "updated_at" : str, + "user" : { + "accepted_tos" : True, + "bio" : str, + "first_name" : "Dave", + "id" : "uMJXuywXLiU", + "instagram_username": "just_midwest_rock", + "last_name" : "Hoefler", + "location" : None, + "name" : "Dave Hoefler", + "portfolio_url" : None, + "total_collections" : int, + "total_likes" : int, + "total_photos" : int, + "twitter_username" : None, + "updated_at" : str, + "username" : "davehoefler", + }, + "views" : int, + "width" : 4480, +}, + +{ + "#url" : "https://unsplash.com/@davehoefler", + "#category": ("", "unsplash", "user"), + "#class" : unsplash.UnsplashUserExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://unsplash.com/@davehoefler/likes", + "#category": ("", "unsplash", "favorite"), + "#class" : unsplash.UnsplashFavoriteExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://unsplash.com/collections/3178572/winter", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, + + "collection_id" : "3178572", + "collection_title": "winter", +}, + +{ + "#url" : "https://unsplash.com/collections/3178572/", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, +}, + +{ + "#url" : "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, +}, + +{ + "#url" : "https://unsplash.com/s/photos/hair-style", + "#category": ("", "unsplash", "search"), + "#class" : unsplash.UnsplashSearchExtractor, + "#pattern" : r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +) diff --git a/test/results/uploadir.py b/test/results/uploadir.py new file mode 100644 index 00000000..18d6a204 --- /dev/null +++ b/test/results/uploadir.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import uploadir + + +__tests__ = ( +{ + "#url" : "https://uploadir.com/u/rd3t46ry", + "#comment" : "image", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/u/rd3t46ry", + "#count" : 1, + + "extension": "jpg", + "filename" : "Chloe and Rachel 4K jpg", + "id" : "rd3t46ry", +}, + +{ + "#url" : "https://uploadir.com/uploads/gxe8ti9v/downloads/new", + "#comment" : "archive", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/uploads/gxe8ti9v/downloads", + "#count" : 1, + + "extension": "zip", + "filename" : "NYAN-Mods-Pack#1", + "id" : "gxe8ti9v", +}, + +{ + "#url" : "https://uploadir.com/u/fllda6xl", + "#comment" : "utf-8 filename", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/u/fllda6xl", + "#count" : 1, + + "extension": "png", + "filename" : "_圖片_🖼_image_", + "id" : "fllda6xl", +}, + +{ + "#url" : "https://uploadir.com/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, +}, + +{ + "#url" : "https://uploadir.com/user/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, +}, + +) diff --git a/test/results/vanillarock.py b/test/results/vanillarock.py new file mode 100644 index 00000000..f0cf3e1c --- /dev/null +++ b/test/results/vanillarock.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vanillarock + + +__tests__ = ( +{ + "#url" : "https://vanilla-rock.com/mizuhashi_parsee-5", + "#category": ("", "vanillarock", "post"), + "#class" : vanillarock.VanillarockPostExtractor, + "#sha1_url" : "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", + "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", +}, + +{ + "#url" : "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", + "#category": ("", "vanillarock", "tag"), + "#class" : vanillarock.VanillarockTagExtractor, + "#pattern" : vanillarock.VanillarockPostExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", + "#category": ("", "vanillarock", "tag"), + "#class" : vanillarock.VanillarockTagExtractor, + "#pattern" : vanillarock.VanillarockPostExtractor.pattern, + "#count" : ">= 5", +}, + +) diff --git a/test/results/vidyart2.py b/test/results/vidyart2.py new file mode 100644 index 00000000..6c11dcbd --- /dev/null +++ b/test/results/vidyart2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "vidyart2", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, +}, + +{ + "#url" : "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "vidyart2", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, +}, + +{ + "#url" : "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", + "#category": ("gelbooru_v01", "vidyart2", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, +}, + +) diff --git a/test/results/vipergirls.py b/test/results/vipergirls.py new file mode 100644 index 00000000..7f0910d1 --- /dev/null +++ b/test/results/vipergirls.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vipergirls + + +__tests__ = ( +{ + "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, + "#count" : 225, + "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", +}, + +{ + "#url" : "https://vipergirls.to/threads/6858916-Karina/page4", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, + "#count" : 1279, +}, + +{ + "#url" : "https://vipergirls.to/threads/4328304", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, +}, + +{ + "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", + "#category": ("", "vipergirls", "post"), + "#class" : vipergirls.VipergirlsPostExtractor, + "#pattern" : r"https://vipr\.im/\w{12}$", + "#range" : "2-113", + "#count" : 112, + + "id" : "116038081", + "imagecount": "113", + "number" : "116038081", + "thread_id" : "4328304", + "title" : "FemJoy Danica - Simply Beautiful (x112) 3000x4500", +}, + +) diff --git a/test/results/vipr.py b/test/results/vipr.py new file mode 100644 index 00000000..f5e00269 --- /dev/null +++ b/test/results/vipr.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://vipr.im/kcd5jcuhgs3v.html", + "#category": ("imagehost", "vipr", "image"), + "#class" : imagehosts.ViprImageExtractor, + "#sha1_url" : "88f6a3ecbf3356a11ae0868b518c60800e070202", + "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", +}, + +) diff --git a/test/results/vk.py b/test/results/vk.py new file mode 100644 index 00000000..b681bdf9 --- /dev/null +++ b/test/results/vk.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vk +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://vk.com/id398982326", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#pattern" : r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", + "#count" : ">= 35", + + "id" : r"re:\d+", + "user": { + "id" : "398982326", + "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", + "name": "", + "nick": "Dobrov Kurva", + }, +}, + +{ + "#url" : "https://vk.com/cosplayinrussia", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#range" : "15-25", + + "id" : r"re:\d+", + "user": { + "id" : "-165740836", + "info": str, + "name": "cosplayinrussia", + "nick": "Косплей | Cosplay 18+", + }, +}, + +{ + "#url" : "https://vk.com/id76957806", + "#comment" : "photos without width/height (#2535)", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#pattern" : r"https://sun\d+-\d+\.userapi\.com/", + "#range" : "1-9", + "#count" : 9, +}, + +{ + "#url" : "https://m.vk.com/albums398982326", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://www.vk.com/id398982326?profile=1", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://vk.com/albums-165740836", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://vk.com/album-165740836_281339889", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://vk.com/album-53775183_00", + "#comment" : "'Access denied' (#2556)", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://vk.com/album232175027_00", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://vk.com/tag304303884", + "#category": ("", "vk", "tagged"), + "#class" : vk.VkTaggedExtractor, + "#count" : 44, +}, + +) diff --git a/test/results/vsco.py b/test/results/vsco.py new file mode 100644 index 00000000..16f501d9 --- /dev/null +++ b/test/results/vsco.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vsco + + +__tests__ = ( +{ + "#url" : "https://vsco.co/missuri/gallery", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, + "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range" : "1-80", + "#count" : 80, +}, + +{ + "#url" : "https://vsco.co/missuri/images/1", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, +}, + +{ + "#url" : "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, +}, + +{ + "#url" : "https://vsco.co/vsco/collection/1", + "#category": ("", "vsco", "collection"), + "#class" : vsco.VscoCollectionExtractor, + "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range" : "1-80", + "#count" : 80, +}, + +{ + "#url" : "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", + "#category": ("", "vsco", "image"), + "#class" : vsco.VscoImageExtractor, + "#sha1_url" : "a45f9712325b42742324b330c348b72477996031", + "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", + + "id" : "5d34b93ef632433030707ce2", + "user" : "erenyildiz", + "grid" : "erenyildiz", + "meta" : dict, + "tags" : list, + "date" : "dt:2019-07-21 19:12:11", + "video" : False, + "width" : 1537, + "height" : 1537, + "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", +}, + +{ + "#url" : "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", + "#category": ("", "vsco", "image"), + "#class" : vsco.VscoImageExtractor, + "#sha1_url" : "08e7eef3301756ce81206c0b47c1e9373756a74a", + "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", + + "video": True, +}, + +) diff --git a/test/results/wallhaven.py b/test/results/wallhaven.py new file mode 100644 index 00000000..47a8ba77 --- /dev/null +++ b/test/results/wallhaven.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wallhaven + + +__tests__ = ( +{ + "#url" : "https://wallhaven.cc/search?q=touhou", + "#category": ("", "wallhaven", "search"), + "#class" : wallhaven.WallhavenSearchExtractor, +}, + +{ + "#url" : "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", + "#category": ("", "wallhaven", "search"), + "#class" : wallhaven.WallhavenSearchExtractor, + "#pattern" : r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#count" : "<= 30", +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/favorites/74", + "#category": ("", "wallhaven", "collection"), + "#class" : wallhaven.WallhavenCollectionExtractor, + "#count" : ">= 50", +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/", + "#category": ("", "wallhaven", "user"), + "#class" : wallhaven.WallhavenUserExtractor, +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/favorites", + "#category": ("", "wallhaven", "collections"), + "#class" : wallhaven.WallhavenCollectionsExtractor, + "#pattern" : wallhaven.WallhavenCollectionExtractor.pattern, + "#count" : 4, +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/uploads", + "#category": ("", "wallhaven", "uploads"), + "#class" : wallhaven.WallhavenUploadsExtractor, + "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://wallhaven.cc/w/01w334", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, + "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", + "#sha1_content": "497212679383a465da1e35bd75873240435085a2", + + "id" : "01w334", + "width" : 1920, + "height" : 1200, + "resolution" : "1920x1200", + "ratio" : "1.6", + "colors" : list, + "tags" : list, + "file_size" : 278799, + "file_type" : "image/jpeg", + "purity" : "sfw", + "short_url" : "https://whvn.cc/01w334", + "source" : str, + "uploader" : { + "group" : "Owner/Developer", + "username": "AksumkA", + }, + "date" : "dt:2014-08-31 06:17:19", + "wh_category": "anime", + "views" : int, + "favorites" : int, +}, + +{ + "#url" : "https://wallhaven.cc/w/dge6v3", + "#comment" : "NSFW", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, + "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", +}, + +{ + "#url" : "https://whvn.cc/01w334", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, +}, + +{ + "#url" : "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, +}, + +) diff --git a/test/results/wallpapercave.py b/test/results/wallpapercave.py new file mode 100644 index 00000000..f1f191ce --- /dev/null +++ b/test/results/wallpapercave.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wallpapercave + + +__tests__ = ( +{ + "#url" : "https://wallpapercave.com/w/wp10270355", + "#category": ("", "wallpapercave", "image"), + "#class" : wallpapercave.WallpapercaveImageExtractor, + "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", +}, + +) diff --git a/test/results/warosu.py b/test/results/warosu.py new file mode 100644 index 00000000..c9273de7 --- /dev/null +++ b/test/results/warosu.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import warosu + + +__tests__ = ( +{ + "#url" : "https://warosu.org/jp/thread/16656025", + "#category": ("", "warosu", "thread"), + "#class" : warosu.WarosuThreadExtractor, + "#sha1_url" : "889d57246ed67e491e5b8f7f124e50ea7991e770", + "#sha1_metadata": "c00ea4c5460c5986994f17bb8416826d42ca57c0", +}, + +{ + "#url" : "https://warosu.org/jp/thread/16658073", + "#category": ("", "warosu", "thread"), + "#class" : warosu.WarosuThreadExtractor, + "#sha1_url" : "4500cf3184b067424fd9883249bd543c905fbecd", + "#sha1_metadata": "7534edf4ec51891dbf44d775b73fbbefd52eec71", + "#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", +}, + +) diff --git a/test/results/weasyl.py b/test/results/weasyl.py new file mode 100644 index 00000000..1b41a6c5 --- /dev/null +++ b/test/results/weasyl.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import weasyl + + +__tests__ = ( +{ + "#url" : "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class" : weasyl.WeasylSubmissionExtractor, + "#pattern" : "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", + + "comments" : int, + "date" : "dt:2012-04-20 00:38:04", + "description" : """<p>(flex)</p> +""", + "favorites" : int, + "folder_name" : "Wesley Stuff", + "folderid" : 2081, + "friends_only": False, + "owner" : "Fiz", + "owner_login" : "fiz", + "rating" : "general", + "submitid" : 2031, + "subtype" : "visual", + "tags" : list, + "title" : "A Wesley!", + "type" : "submission", + "views" : int, +}, + +{ + "#url" : "https://www.weasyl.com/submission/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class" : weasyl.WeasylSubmissionExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/~tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, + "#count" : ">= 200", +}, + +{ + "#url" : "https://www.weasyl.com/submissions/tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/~aro~so", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/submissions/tanidareal?folderid=7403", + "#category": ("", "weasyl", "folder"), + "#class" : weasyl.WeasylFolderExtractor, + "#count" : ">= 12", +}, + +{ + "#url" : "https://www.weasyl.com/journal/17647/bbcode", + "#category": ("", "weasyl", "journal"), + "#class" : weasyl.WeasylJournalExtractor, + + "title" : "BBCode", + "date" : "dt:2013-09-19 23:11:23", + "content": """<p><a>javascript:alert(42);</a></p> + +<p>No more of that!</p> +""", +}, + +{ + "#url" : "https://www.weasyl.com/journals/charmander", + "#category": ("", "weasyl", "journals"), + "#class" : weasyl.WeasylJournalsExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.weasyl.com/favorites?userid=184616&feature=submit", + "#category": ("", "weasyl", "favorite"), + "#class" : weasyl.WeasylFavoriteExtractor, + "#count" : ">= 5", +}, + +) diff --git a/test/results/webmshare.py b/test/results/webmshare.py new file mode 100644 index 00000000..007a12b4 --- /dev/null +++ b/test/results/webmshare.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import webmshare + + +__tests__ = ( +{ + "#url" : "https://webmshare.com/O9mWY", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, + + "date" : "dt:2022-12-04 00:00:00", + "extension": "webm", + "filename" : "O9mWY", + "height" : 568, + "id" : "O9mWY", + "thumb" : "https://s1.webmshare.com/t/O9mWY.jpg", + "title" : "Yeah buddy over here", + "url" : "https://s1.webmshare.com/O9mWY.webm", + "views" : int, + "width" : 320, +}, + +{ + "#url" : "https://s1.webmshare.com/zBGAg.webm", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, + + "date" : "dt:2018-12-07 00:00:00", + "height": 1080, + "id" : "zBGAg", + "thumb" : "https://s1.webmshare.com/t/zBGAg.jpg", + "title" : "", + "url" : "https://s1.webmshare.com/zBGAg.webm", + "views" : int, + "width" : 1920, +}, + +{ + "#url" : "https://webmshare.com/play/zBGAg", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, +}, + +{ + "#url" : "https://webmshare.com/download-webm/zBGAg", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, +}, + +) diff --git a/test/results/webtoons.py b/test/results/webtoons.py new file mode 100644 index 00000000..d2a177fd --- /dev/null +++ b/test/results/webtoons.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import webtoons +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", + "#category": ("", "webtoons", "episode"), + "#class" : webtoons.WebtoonsEpisodeExtractor, + "#count" : 5, + "#sha1_url" : "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "#sha1_content": [ + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ], +}, + +{ + "#url" : "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", + "#category": ("", "webtoons", "episode"), + "#class" : webtoons.WebtoonsEpisodeExtractor, + "#exception": exception.NotFoundError, + + "comic" : "punderworld", + "description": str, + "episode" : "36", + "episode_no" : "40", + "genre" : "challenge", + "title" : r"re:^Punderworld - .+", + "title_no" : "312584", +}, + +{ + "#url" : "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", + "#comment" : "english", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#pattern" : webtoons.WebtoonsEpisodeExtractor.pattern, + "#range" : "1-15", + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=3", + "#comment" : "french", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", + "#comment" : "(#820)", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#count" : ">= 18", +}, + +{ + "#url" : "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", + "#comment" : "(#1643)", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, +}, + +) diff --git a/test/results/weibo.py b/test/results/weibo.py new file mode 100644 index 00000000..f98b7fd9 --- /dev/null +++ b/test/results/weibo.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import weibo +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, + "#pattern" : r"^https://weibo\.com/u/1758989602\?tabtype=feed$", +}, + +{ + "#url" : "https://weibo.com/u/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://weibo.com/p/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/profile/2314621010", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://www.weibo.com/p/1003062314621010/home", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=home", + "#category": ("", "weibo", "home"), + "#class" : weibo.WeiboHomeExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/zhouyuxi77?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1", + + "status": {"user": { + "id" : 7488709788, +}}, +}, + +{ + "#url" : "https://www.weibo.com/n/周于希Sally?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1", + + "status": {"user": { + "id" : 7488709788, +}}, +}, + +{ + "#url" : "https://weibo.com/u/7500315942?tabtype=feed", + "#comment" : "deleted (#2521)", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=video", + "#category": ("", "weibo", "videos"), + "#class" : weibo.WeiboVideosExtractor, + "#pattern" : r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=newVideo", + "#category": ("", "weibo", "newvideo"), + "#class" : weibo.WeiboNewvideoExtractor, + "#pattern" : r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=article", + "#category": ("", "weibo", "article"), + "#class" : weibo.WeiboArticleExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=album", + "#category": ("", "weibo", "album"), + "#class" : weibo.WeiboAlbumExtractor, + "#pattern" : r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", + "#range" : "1-3", + "#count" : 3, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4323047042991618", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", + + "status": { + "count": 1, + "date" : "dt:2018-12-30 13:56:36", + }, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", +}, + +{ + "#url" : "https://m.weibo.cn/status/4268682979207023", + "#comment" : "unavailable video (#427)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://weibo.com/3314883543/Iy7fj4qVg", + "#comment" : "non-numeric status ID (#664)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4600272267522211", + "#comment" : "original retweets (#1542)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#options" : {"retweets": "original"}, + + "status": {"id": 4600167083287033}, +}, + +{ + "#url" : "https://weibo.com/5643044717/KkuDZ4jAA", + "#comment" : "type == livephoto (#2146)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://video\.weibo\.com/media/play\?livephoto=https%3A%2F%2Fus.sinaimg.cn%2F\w+\.mov", + "#range" : "2,4,6", +}, + +{ + "#url" : "https://weibo.com/1758989602/LvBhm5DiP", + "#comment" : "type == gif", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", +}, + +{ + "#url" : "https://weibo.com/2909128931/4409545658754086", + "#comment" : "missing 'playback_list' (#2792)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#count" : 10, +}, + +{ + "#url" : "https://weibo.com/1501933722/4142890299009993", + "#comment" : "empty 'playback_list' (#3301)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", + "#count" : 1, +}, + +{ + "#url" : "https://weibo.com/2427303621/MxojLlLgQ", + "#comment" : "mix_media_info (#3793)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#count" : 9, +}, + +{ + "#url" : "https://m.weibo.cn/status/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/5746766133/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +) diff --git a/test/results/wikiart.py b/test/results/wikiart.py new file mode 100644 index 00000000..fef4bad5 --- /dev/null +++ b/test/results/wikiart.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikiart + + +__tests__ = ( +{ + "#url" : "https://www.wikiart.org/en/thomas-cole", + "#category": ("", "wikiart", "artist"), + "#class" : wikiart.WikiartArtistExtractor, + "#sha1_url" : "6844f207a5063c499fc1d5651b03127bc4fe2f73", + "#sha1_metadata": "09230b5f504697119e267349bf92487e657a7384", +}, + +{ + "#url" : "https://www.wikiart.org/en/thomas-cole/the-departure-1838", + "#category": ("", "wikiart", "image"), + "#class" : wikiart.WikiartImageExtractor, + "#sha1_url" : "976cc2545f308a650b5dbb35c29d3cee0f4673b3", + "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", +}, + +{ + "#url" : "https://www.wikiart.org/en/huang-shen/summer", + "#comment" : "no year or '-' in slug", + "#category": ("", "wikiart", "image"), + "#class" : wikiart.WikiartImageExtractor, + "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", +}, + +{ + "#url" : "https://www.wikiart.org/en/paintings-by-media/grisaille", + "#category": ("", "wikiart", "artworks"), + "#class" : wikiart.WikiartArtworksExtractor, + "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", +}, + +{ + "#url" : "https://www.wikiart.org/en/artists-by-century/12", + "#category": ("", "wikiart", "artists"), + "#class" : wikiart.WikiartArtistsExtractor, + "#pattern" : wikiart.WikiartArtistExtractor.pattern, + "#count" : ">= 8", +}, + +) diff --git a/test/results/wikieat.py b/test/results/wikieat.py new file mode 100644 index 00000000..f3c0d7f4 --- /dev/null +++ b/test/results/wikieat.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://wikieat.club/cel/res/25321.html", + "#category": ("vichan", "wikieat", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+", + "#count" : ">= 200", +}, + +{ + "#url" : "https://wikieat.club/cel/index.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://wikieat.club/cel/catalog.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://wikieat.club/cel/2.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/wikifeet.py b/test/results/wikifeet.py new file mode 100644 index 00000000..dade42e7 --- /dev/null +++ b/test/results/wikifeet.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikifeet + + +__tests__ = ( +{ + "#url" : "https://www.wikifeet.com/Madison_Beer", + "#category": ("", "wikifeet", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", + "#count" : ">= 352", + + "celeb" : "Madison_Beer", + "celebrity" : "Madison Beer", + "birthday" : "dt:1999-03-05 00:00:00", + "birthplace": "United States", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "9 US", + "type" : "women", + "tags" : list, +}, + +{ + "#url" : "https://men.wikifeet.com/Chris_Hemsworth", + "#category": ("", "wikifeet", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", + "#count" : ">= 860", + + "celeb" : "Chris_Hemsworth", + "celebrity" : "Chris Hemsworth", + "birthday" : "dt:1983-08-11 00:00:00", + "birthplace": "Australia", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "12.5 US", + "type" : "men", + "tags" : list, +}, + +) diff --git a/test/results/wikifeetx.py b/test/results/wikifeetx.py new file mode 100644 index 00000000..a3d2be6e --- /dev/null +++ b/test/results/wikifeetx.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikifeet + + +__tests__ = ( +{ + "#url" : "https://www.wikifeetx.com/Tifa_Quinn", + "#category": ("", "wikifeetx", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", + "#count" : ">= 9", + + "celeb" : "Tifa_Quinn", + "celebrity" : "Tifa Quinn", + "birthday" : "[NOT SET]", + "birthplace": "United States", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "4 US", + "type" : "women", + "tags" : list, +}, + +) diff --git a/test/results/windsorstore.py b/test/results/windsorstore.py new file mode 100644 index 00000000..c7cfd698 --- /dev/null +++ b/test/results/windsorstore.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.windsorstore.com/collections/dresses-ball-gowns", + "#category": ("shopify", "windsorstore", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", + "#category": ("shopify", "windsorstore", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/xbooru.py b/test/results/xbooru.py new file mode 100644 index 00000000..11160b42 --- /dev/null +++ b/test/results/xbooru.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", + "#category": ("gelbooru_v02", "xbooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : 11, +}, + +{ + "#url" : "https://xbooru.com/index.php?page=pool&s=show&id=757", + "#category": ("gelbooru_v02", "xbooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 5, + "#sha1_url": "ceeac56c179ec72301bd0b6add6355a138fdea01", +}, + +{ + "#url" : "https://xbooru.com/index.php?page=favorites&s=view&id=45206", + "#category": ("gelbooru_v02", "xbooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://xbooru.com/index.php?page=post&s=view&id=1025649", + "#category": ("gelbooru_v02", "xbooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#pattern" : r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", + "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", +}, + +) diff --git a/test/results/xbunkr.py b/test/results/xbunkr.py new file mode 100644 index 00000000..a188d2e6 --- /dev/null +++ b/test/results/xbunkr.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lolisafe + + +__tests__ = ( +{ + "#url" : "https://xbunkr.com/a/TA0bu3F4", + "#category": ("lolisafe", "xbunkr", "album"), + "#class" : lolisafe.LolisafeAlbumExtractor, + "#pattern" : r"https://media\.xbunkr\.com/[^.]+\.\w+", + "#count" : 861, + + "album_id" : "TA0bu3F4", + "album_name": "Hannahowo Onlyfans Photos", +}, + +{ + "#url" : "https://xbunkr.com/a/GNQc2I5d", + "#category": ("lolisafe", "xbunkr", "album"), + "#class" : lolisafe.LolisafeAlbumExtractor, +}, + +) diff --git a/test/results/xhamster.py b/test/results/xhamster.py new file mode 100644 index 00000000..93436d06 --- /dev/null +++ b/test/results/xhamster.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import xhamster + + +__tests__ = ( +{ + "#url" : "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, + "#pattern" : r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", + "#count" : ">= 144", + + "comments": int, + "count" : int, + "favorite": bool, + "id" : int, + "num" : int, + "height" : int, + "width" : int, + "imageURL": str, + "pageURL" : str, + "thumbURL": str, + "gallery" : { + "date" : "dt:2019-04-16 00:07:31", + "description": "", + "dislikes" : int, + "id" : 11748968, + "likes" : int, + "tags" : ["NON-Porn"], + "thumbnail" : str, + "title" : "Make the world better.", + "views" : int, + }, + "user" : { + "id" : 16874672, + "name" : "Anonymousrants", + "retired" : bool, + "subscribers": int, + "url" : "https://xhamster.com/users/anonymousrants", + "verified" : bool, + }, +}, + +{ + "#url" : "https://jp.xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, + "#pattern" : r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", + "#count" : ">= 144", +}, + +{ + "#url" : "https://xhamster.com/photos/gallery/make-the-world-better-11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.one/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.desi/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://en.xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.porncache.net/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.com/users/goldenpalomino/photos", + "#category": ("", "xhamster", "user"), + "#class" : xhamster.XhamsterUserExtractor, + "#pattern" : xhamster.XhamsterGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://xhamster.com/users/nickname68", + "#category": ("", "xhamster", "user"), + "#class" : xhamster.XhamsterUserExtractor, +}, + +) diff --git a/test/results/xvideos.py b/test/results/xvideos.py new file mode 100644 index 00000000..d7a682c0 --- /dev/null +++ b/test/results/xvideos.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import xvideos + + +__tests__ = ( +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, + "#pattern" : r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", + "#count" : 8, + + "gallery": { + "id" : 751031, + "title": "Random Stuff", + "tags" : list, + }, + "user" : { + "id" : 20245371, + "name" : "pervertedcouple", + "display" : "Pervertedcouple", + "sex" : "Woman", + "description": str, + }, +}, + +{ + "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, +}, + +{ + "#url" : "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, +}, + +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class" : xvideos.XvideosUserExtractor, + "#sha1_url" : "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", + "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", +}, + +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", + "#category": ("", "xvideos", "user"), + "#class" : xvideos.XvideosUserExtractor, +}, + +) diff --git a/test/results/yandere.py b/test/results/yandere.py new file mode 100644 index 00000000..3b4d9a96 --- /dev/null +++ b/test/results/yandere.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://yande.re/post/show/51824", + "#category": ("moebooru", "yandere", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + + "tags_artist" : "sasaki_tamaru", + "tags_circle" : "softhouse_chara", + "tags_copyright": "ouzoku", + "tags_general" : str, +}, + +{ + "#url" : "https://yande.re/post/show/993156", + "#category": ("moebooru", "yandere", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"notes": True}, + "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", + + "notes": [ + { + "id" : 7096, + "x" : 90, + "y" : 626, + "width" : 283, + "height": 529, + "body" : "Please keep this as a secret for me!!", + }, + { + "id" : 7095, + "x" : 900, + "y" : 438, + "width" : 314, + "height": 588, + "body" : "The facts that I love playing games", + }, + ], +}, + +{ + "#url" : "https://yande.re/post?tags=ouzoku+armor", + "#category": ("moebooru", "yandere", "tag"), + "#class" : moebooru.MoebooruTagExtractor, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", +}, + +{ + "#url" : "https://yande.re/pool/show/318", + "#category": ("moebooru", "yandere", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, + "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", +}, + +{ + "#url" : "https://yande.re/post/popular_by_month?month=6&year=2014", + "#category": ("moebooru", "yandere", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, + "#count" : 40, +}, + +{ + "#url" : "https://yande.re/post/popular_recent", + "#category": ("moebooru", "yandere", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/ytdl.py b/test/results/ytdl.py new file mode 100644 index 00000000..1aecee91 --- /dev/null +++ b/test/results/ytdl.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import ytdl + + +__tests__ = ( +{ + "#url" : "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", + "#category": ("", "ytdl", "Youtube"), + "#class" : ytdl.YoutubeDLExtractor, +}, + +) diff --git a/test/results/zerochan.py b/test/results/zerochan.py new file mode 100644 index 00000000..67b9da21 --- /dev/null +++ b/test/results/zerochan.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import zerochan + + +__tests__ = ( +{ + "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class" : zerochan.ZerochanTagExtractor, + "#pattern" : r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "#count" : "> 24", + + "extension" : r"re:jpg|png", + "file_url" : r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "filename" : r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", + "height" : r"re:^\d+$", + "id" : r"re:^\d+$", + "name" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "search_tags": "Perth (Kantai Collection)", + "size" : r"re:^\d+k$", + "width" : r"re:^\d+$", +}, + +{ + "#url" : "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class" : zerochan.ZerochanImageExtractor, + "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + + "author" : "YeFan 葉凡", + "date" : "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height" : 1366, + "id" : 2920445, + "path" : [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size" : 1975296, + "tags" : [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Blonde Hair", + "Theme:Braids", + "Theme:Coat", + "Theme:Female", + "Theme:Firefighter Outfit", + "Theme:Group", + "Theme:Long Sleeves", + "Theme:Personification", + "Theme:Pins", + "Theme:Ribbon", + "Theme:Shirt", + "Theme:Short Hair", + ], + "uploader": "YukinoTokisaki", + "width" : 1920, +}, + +) From a453335a9fbb6374eac51a691e70d28a83514b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 11 Sep 2023 16:30:55 +0200 Subject: [PATCH 34/53] remove test results in extractor modules and add generic example URLs --- gallery_dl/extractor/2chan.py | 24 +- gallery_dl/extractor/2chen.py | 30 +- gallery_dl/extractor/35photo.py | 44 +-- gallery_dl/extractor/3dbooru.py | 27 +- gallery_dl/extractor/4chan.py | 19 +- gallery_dl/extractor/4chanarchives.py | 26 +- gallery_dl/extractor/500px.py | 69 +---- gallery_dl/extractor/8chan.py | 53 +--- gallery_dl/extractor/8muses.py | 46 +-- gallery_dl/extractor/adultempire.py | 14 +- gallery_dl/extractor/architizer.py | 27 +- gallery_dl/extractor/artstation.py | 85 +----- gallery_dl/extractor/aryion.py | 52 +--- gallery_dl/extractor/bbc.py | 25 +- gallery_dl/extractor/behance.py | 52 +--- gallery_dl/extractor/blogger.py | 82 +---- gallery_dl/extractor/bunkr.py | 48 +-- gallery_dl/extractor/catbox.py | 27 +- gallery_dl/extractor/comicvine.py | 16 +- gallery_dl/extractor/common.py | 2 +- gallery_dl/extractor/cyberdrop.py | 27 +- gallery_dl/extractor/danbooru.py | 82 +---- gallery_dl/extractor/desktopography.py | 6 +- gallery_dl/extractor/deviantart.py | 387 ++---------------------- gallery_dl/extractor/directlink.py | 33 +- gallery_dl/extractor/dynastyscans.py | 34 +-- gallery_dl/extractor/e621.py | 135 +-------- gallery_dl/extractor/erome.py | 34 +-- gallery_dl/extractor/exhentai.py | 82 +---- gallery_dl/extractor/fallenangels.py | 30 +- gallery_dl/extractor/fanbox.py | 72 +---- gallery_dl/extractor/fanleaks.py | 50 +-- gallery_dl/extractor/fantia.py | 60 +--- gallery_dl/extractor/fapachi.py | 27 +- gallery_dl/extractor/fapello.py | 53 +--- gallery_dl/extractor/flickr.py | 66 +--- gallery_dl/extractor/foolfuuka.py | 74 +---- gallery_dl/extractor/foolslide.py | 26 +- gallery_dl/extractor/furaffinity.py | 118 +------- gallery_dl/extractor/fuskator.py | 27 +- gallery_dl/extractor/gelbooru.py | 83 +---- gallery_dl/extractor/gelbooru_v01.py | 70 +---- gallery_dl/extractor/gelbooru_v02.py | 178 +---------- gallery_dl/extractor/generic.py | 26 +- gallery_dl/extractor/gofile.py | 46 +-- gallery_dl/extractor/hbrowse.py | 11 +- gallery_dl/extractor/hentai2read.py | 54 +--- gallery_dl/extractor/hentaicosplays.py | 31 +- gallery_dl/extractor/hentaifoundry.py | 91 +----- gallery_dl/extractor/hentaifox.py | 52 +--- gallery_dl/extractor/hentaihand.py | 32 +- gallery_dl/extractor/hentaihere.py | 48 +-- gallery_dl/extractor/hiperdex.py | 61 +--- gallery_dl/extractor/hitomi.py | 54 +--- gallery_dl/extractor/hotleak.py | 62 +--- gallery_dl/extractor/idolcomplex.py | 33 +- gallery_dl/extractor/imagebam.py | 42 +-- gallery_dl/extractor/imagechest.py | 24 +- gallery_dl/extractor/imagefap.py | 107 +------ gallery_dl/extractor/imagehosts.py | 132 +------- gallery_dl/extractor/imgbb.py | 43 +-- gallery_dl/extractor/imgbox.py | 30 +- gallery_dl/extractor/imgth.py | 21 +- gallery_dl/extractor/imgur.py | 209 +------------ gallery_dl/extractor/inkbunny.py | 113 +------ gallery_dl/extractor/instagram.py | 164 +--------- gallery_dl/extractor/issuu.py | 31 +- gallery_dl/extractor/itaku.py | 66 +--- gallery_dl/extractor/itchio.py | 23 +- gallery_dl/extractor/jpgfish.py | 55 +--- gallery_dl/extractor/jschan.py | 17 +- gallery_dl/extractor/kabeuchi.py | 13 +- gallery_dl/extractor/keenspot.py | 28 +- gallery_dl/extractor/kemonoparty.py | 150 +-------- gallery_dl/extractor/khinsider.py | 25 +- gallery_dl/extractor/komikcast.py | 23 +- gallery_dl/extractor/lensdump.py | 33 +- gallery_dl/extractor/lexica.py | 32 +- gallery_dl/extractor/lightroom.py | 19 +- gallery_dl/extractor/livedoor.py | 44 +-- gallery_dl/extractor/lolisafe.py | 14 +- gallery_dl/extractor/luscious.py | 78 +---- gallery_dl/extractor/lynxchan.py | 36 +-- gallery_dl/extractor/mangadex.py | 67 +--- gallery_dl/extractor/mangafox.py | 40 +-- gallery_dl/extractor/mangahere.py | 32 +- gallery_dl/extractor/mangakakalot.py | 19 +- gallery_dl/extractor/manganelo.py | 36 +-- gallery_dl/extractor/mangapark.py | 70 +---- gallery_dl/extractor/mangaread.py | 96 +----- gallery_dl/extractor/mangasee.py | 89 +----- gallery_dl/extractor/mangoxo.py | 28 +- gallery_dl/extractor/mastodon.py | 56 +--- gallery_dl/extractor/misskey.py | 50 +-- gallery_dl/extractor/moebooru.py | 125 ++------ gallery_dl/extractor/myhentaigallery.py | 19 +- gallery_dl/extractor/myportfolio.py | 22 +- gallery_dl/extractor/naver.py | 27 +- gallery_dl/extractor/naverwebtoon.py | 64 +--- gallery_dl/extractor/newgrounds.py | 200 +----------- gallery_dl/extractor/nhentai.py | 46 +-- gallery_dl/extractor/nijie.py | 133 +------- gallery_dl/extractor/nitter.py | 183 +---------- gallery_dl/extractor/nozomi.py | 63 +--- gallery_dl/extractor/nsfwalbum.py | 8 +- gallery_dl/extractor/nudecollect.py | 63 +--- gallery_dl/extractor/oauth.py | 7 + gallery_dl/extractor/paheal.py | 85 +----- gallery_dl/extractor/patreon.py | 50 +-- gallery_dl/extractor/philomena.py | 109 +------ gallery_dl/extractor/photobucket.py | 41 +-- gallery_dl/extractor/photovogue.py | 33 +- gallery_dl/extractor/picarto.py | 9 +- gallery_dl/extractor/piczel.py | 34 +-- gallery_dl/extractor/pillowfort.py | 72 +---- gallery_dl/extractor/pinterest.py | 89 +----- gallery_dl/extractor/pixiv.py | 270 ++--------------- gallery_dl/extractor/pixnet.py | 40 +-- gallery_dl/extractor/plurk.py | 16 +- gallery_dl/extractor/poipiku.py | 59 +--- gallery_dl/extractor/pornhub.py | 64 +--- gallery_dl/extractor/pornpics.py | 58 +--- gallery_dl/extractor/pururin.py | 32 +- gallery_dl/extractor/reactor.py | 90 +----- gallery_dl/extractor/readcomiconline.py | 19 +- gallery_dl/extractor/recursive.py | 6 +- gallery_dl/extractor/reddit.py | 116 +------ gallery_dl/extractor/redgifs.py | 81 +---- gallery_dl/extractor/rule34us.py | 34 +-- gallery_dl/extractor/sankaku.py | 88 +----- gallery_dl/extractor/sankakucomplex.py | 38 +-- gallery_dl/extractor/seiga.py | 48 +-- gallery_dl/extractor/senmanga.py | 49 +-- gallery_dl/extractor/sexcom.py | 64 +--- gallery_dl/extractor/shimmie2.py | 141 +-------- gallery_dl/extractor/shopify.py | 48 +-- gallery_dl/extractor/simplyhentai.py | 44 +-- gallery_dl/extractor/skeb.py | 63 +--- gallery_dl/extractor/slickpic.py | 28 +- gallery_dl/extractor/slideshare.py | 43 +-- gallery_dl/extractor/smugmug.py | 53 +--- gallery_dl/extractor/soundgasm.py | 38 +-- gallery_dl/extractor/speakerdeck.py | 14 +- gallery_dl/extractor/subscribestar.py | 57 +--- gallery_dl/extractor/szurubooru.py | 24 +- gallery_dl/extractor/tapas.py | 59 +--- gallery_dl/extractor/tcbscans.py | 44 +-- gallery_dl/extractor/telegraph.py | 67 +--- gallery_dl/extractor/test.py | 8 +- gallery_dl/extractor/toyhouse.py | 57 +--- gallery_dl/extractor/tsumino.py | 48 +-- gallery_dl/extractor/tumblr.py | 145 +-------- gallery_dl/extractor/tumblrgallery.py | 13 +- gallery_dl/extractor/twibooru.py | 72 +---- gallery_dl/extractor/twitter.py | 320 ++------------------ gallery_dl/extractor/unsplash.py | 98 +----- gallery_dl/extractor/uploadir.py | 37 +-- gallery_dl/extractor/urlshortener.py | 15 +- gallery_dl/extractor/vanillarock.py | 20 +- gallery_dl/extractor/vichan.py | 53 +--- gallery_dl/extractor/vipergirls.py | 28 +- gallery_dl/extractor/vk.py | 57 +--- gallery_dl/extractor/vsco.py | 41 +-- gallery_dl/extractor/wallhaven.py | 64 +--- gallery_dl/extractor/wallpapercave.py | 4 +- gallery_dl/extractor/warosu.py | 14 +- gallery_dl/extractor/weasyl.py | 56 +--- gallery_dl/extractor/webmshare.py | 33 +- gallery_dl/extractor/webtoons.py | 25 +- gallery_dl/extractor/weibo.py | 111 +------ gallery_dl/extractor/wikiart.py | 27 +- gallery_dl/extractor/wikifeet.py | 57 +--- gallery_dl/extractor/xhamster.py | 60 +--- gallery_dl/extractor/xvideos.py | 35 +-- gallery_dl/extractor/ytdl.py | 4 +- gallery_dl/extractor/zerochan.py | 53 +--- 176 files changed, 656 insertions(+), 9554 deletions(-) diff --git a/gallery_dl/extractor/2chan.py b/gallery_dl/extractor/2chan.py index 92ea6caf..337ba480 100644 --- a/gallery_dl/extractor/2chan.py +++ b/gallery_dl/extractor/2chan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,26 +20,8 @@ class _2chanThreadExtractor(Extractor): filename_fmt = "{tim}.{extension}" archive_fmt = "{board}_{thread}_{tim}" url_fmt = "https://{server}.2chan.net/{board}/src/{filename}" - pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/]+)/res/(\d+)" - test = ("https://dec.2chan.net/70/res/14565.htm", { - "pattern": r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", - "count": ">= 3", - "keyword": { - "board": "70", - "board_name": "新板提案", - "com": str, - "fsize": r"re:\d+", - "name": "名無し", - "no": r"re:1[45]\d\d\d", - "now": r"re:22/../..\(.\)..:..:..", - "post": "無題", - "server": "dec", - "thread": "14565", - "tim": r"re:^\d{13}$", - "time": r"re:^\d{10}$", - "title": "ﾋﾛｱｶ板" - }, - }) + pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/?#]+)/res/(\d+)" + example = "https://dec.2chan.net/12/res/12345.htm" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index f1426900..0c978897 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -21,26 +21,7 @@ class _2chenThreadExtractor(Extractor): filename_fmt = "{time} {filename}.{extension}" archive_fmt = "{board}_{thread}_{hash}_{time}" pattern = BASE_PATTERN + r"/([^/?#]+)/(\d+)" - test = ( - ("https://sturdychan.help/tv/268929", { - "pattern": r"https://sturdychan\.help/assets/images" - r"/src/\w{40}\.\w+$", - "count": ">= 179", - "keyword": { - "board": "tv", - "date": "type:datetime", - "hash": r"re:[0-9a-f]{40}", - "name": "Anonymous", - "no": r"re:\d+", - "thread": "268929", - "time": int, - "title": "「/ttg/ #118: 🇧🇷 edition」", - "url": str, - }, - }), - ("https://2chen.club/tv/1"), - ("https://2chen.moe/jp/303786"), - ) + example = "https://sturdychan.help/a/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -101,14 +82,7 @@ class _2chenBoardExtractor(Extractor): subcategory = "board" root = "https://sturdychan.help" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/catalog|/?$)" - test = ( - ("https://sturdychan.help/co/", { - "pattern": _2chenThreadExtractor.pattern - }), - ("https://2chen.moe/co"), - ("https://2chen.club/tv"), - ("https://2chen.moe/co/catalog"), - ) + example = "https://sturdychan.help/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/35photo.py b/gallery_dl/extractor/35photo.py index f86691d4..773116ea 100644 --- a/gallery_dl/extractor/35photo.py +++ b/gallery_dl/extractor/35photo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -101,20 +101,7 @@ class _35photoUserExtractor(_35photoExtractor): subcategory = "user" pattern = (r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro" r"/(?!photo_|genre_|tags/|rating/)([^/?#]+)") - test = ( - ("https://35photo.pro/liya", { - "pattern": r"https://([a-z][0-9]\.)?35photo\.pro" - r"/photos_(main|series)/.*\.jpg", - "count": 9, - }), - ("https://35photo.pro/suhoveev", { - # last photo ID (1267028) isn't given as 'photo-id="<id>" - # there are only 23 photos without the last one - "count": ">= 33", - }), - ("https://en.35photo.pro/liya"), - ("https://ru.35photo.pro/liya"), - ) + example = "https://35photo.pro/USER" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -143,11 +130,7 @@ class _35photoTagExtractor(_35photoExtractor): directory_fmt = ("{category}", "Tags", "{search_tag}") archive_fmt = "t{search_tag}_{id}_{num}" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/tags/([^/?#]+)" - test = ("https://35photo.pro/tags/landscape/", { - "range": "1-25", - "count": 25, - "archive": False, - }) + example = "https://35photo.pro/tags/TAG/" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -180,7 +163,7 @@ class _35photoGenreExtractor(_35photoExtractor): directory_fmt = ("{category}", "Genre", "{genre}") archive_fmt = "g{genre_id}_{id}_{num}" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/genre_(\d+)(/new/)?" - test = ("https://35photo.pro/genre_109/",) + example = "https://35photo.pro/genre_12345/" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -212,24 +195,7 @@ class _35photoImageExtractor(_35photoExtractor): """Extractor for individual images from 35photo.pro""" subcategory = "image" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/photo_(\d+)" - test = ("https://35photo.pro/photo_753340/", { - "count": 1, - "keyword": { - "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", - "id" : 753340, - "title" : "Winter walk", - "description": str, - "tags" : list, - "views" : int, - "favorites" : int, - "score" : int, - "type" : 0, - "date" : "15 авг, 2014", - "user" : "liya", - "user_id" : 20415, - "user_name" : "Liya Mirzaeva", - }, - }) + example = "https://35photo.pro/photo_12345/" def __init__(self, match): _35photoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/3dbooru.py b/gallery_dl/extractor/3dbooru.py index e83bca75..c1bc9e5e 100644 --- a/gallery_dl/extractor/3dbooru.py +++ b/gallery_dl/extractor/3dbooru.py @@ -27,10 +27,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor): """Extractor for images from behoimi.org based on search-tags""" pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org/post" r"(?:/(?:index)?)?\?tags=(?P<tags>[^&#]+)") - test = ("http://behoimi.org/post?tags=himekawa_azuru+dress", { - "url": "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", - "content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", - }) + example = "http://behoimi.org/post?tags=TAG" def posts(self): params = {"tags": self.tags} @@ -40,10 +37,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor): class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): """Extractor for image-pools from behoimi.org""" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P<pool>\d+)" - test = ("http://behoimi.org/pool/show/27", { - "url": "da75d2d1475449d5ef0c266cb612683b110a30f2", - "content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", - }) + example = "http://behoimi.org/pool/show/12345" def posts(self): params = {"tags": "pool:" + self.pool_id} @@ -53,17 +47,7 @@ class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor): """Extractor for single images from behoimi.org""" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P<post>\d+)" - test = ("http://behoimi.org/post/show/140852", { - "url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", - "content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", - "options": (("tags", True),), - "keyword": { - "tags_character": "furude_rika", - "tags_copyright": "higurashi_no_naku_koro_ni", - "tags_model": "himekawa_azuru", - "tags_general": str, - }, - }) + example = "http://behoimi.org/post/show/12345" def posts(self): params = {"tags": "id:" + self.post_id} @@ -76,7 +60,4 @@ class _3dbooruPopularExtractor( pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org" r"/post/popular_(?P<scale>by_(?:day|week|month)|recent)" r"(?:\?(?P<query>[^#]*))?") - test = ("http://behoimi.org/post/popular_by_month?month=2&year=2013", { - "pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", - "count": 20, - }) + example = "http://behoimi.org/post/popular_by_month" diff --git a/gallery_dl/extractor/4chan.py b/gallery_dl/extractor/4chan.py index bf9615d0..2db60422 100644 --- a/gallery_dl/extractor/4chan.py +++ b/gallery_dl/extractor/4chan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2019 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,17 +21,7 @@ class _4chanThreadExtractor(Extractor): archive_fmt = "{board}_{thread}_{tim}" pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org" r"/([^/]+)/thread/(\d+)") - test = ( - ("https://boards.4chan.org/tg/thread/15396072/", { - "url": "39082ad166161966d7ba8e37f2173a824eb540f0", - "keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - "content": "20b7b51afa51c9c31a0020a0737b889532c8d7ec", - }), - ("https://boards.4channel.org/tg/thread/15396072/", { - "url": "39082ad166161966d7ba8e37f2173a824eb540f0", - "keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - }), - ) + example = "https://boards.4channel.org/a/thread/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -65,10 +55,7 @@ class _4chanBoardExtractor(Extractor): category = "4chan" subcategory = "board" pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$" - test = ("https://boards.4channel.org/po/", { - "pattern": _4chanThreadExtractor.pattern, - "count": ">= 100", - }) + example = "https://boards.4channel.org/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/4chanarchives.py b/gallery_dl/extractor/4chanarchives.py index 041e6a35..f018d3ec 100644 --- a/gallery_dl/extractor/4chanarchives.py +++ b/gallery_dl/extractor/4chanarchives.py @@ -21,21 +21,7 @@ class _4chanarchivesThreadExtractor(Extractor): filename_fmt = "{no}-{filename}.{extension}" archive_fmt = "{board}_{thread}_{no}" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)/thread/(\d+)" - test = ( - ("https://4chanarchives.com/board/c/thread/2707110", { - "pattern": r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", - "count": 2, - "keyword": { - "board": "c", - "com": str, - "name": "Anonymous", - "no": int, - "thread": "2707110", - "time": r"re:2016-07-1\d \d\d:\d\d:\d\d", - "title": "Ren Kagami from 'Oyako Neburi'", - }, - }), - ) + example = "https://4chanarchives.com/board/a/thread/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -106,15 +92,7 @@ class _4chanarchivesBoardExtractor(Extractor): subcategory = "board" root = "https://4chanarchives.com" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)(?:/(\d+))?/?$" - test = ( - ("https://4chanarchives.com/board/c/", { - "pattern": _4chanarchivesThreadExtractor.pattern, - "range": "1-40", - "count": 40, - }), - ("https://4chanarchives.com/board/c"), - ("https://4chanarchives.com/board/c/10"), - ) + example = "https://4chanarchives.com/board/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index 4d1307e1..a1a9d0d9 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -96,15 +96,7 @@ class _500pxUserExtractor(_500pxExtractor): """Extractor for photos from a user's photostream on 500px.com""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!photo/|liked)(?:p/)?([^/?#]+)/?(?:$|[?#])" - test = ( - ("https://500px.com/p/light_expression_photography", { - "pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", - "range": "1-99", - "count": 99, - }), - ("https://500px.com/light_expression_photography"), - ("https://web.500px.com/light_expression_photography"), - ) + example = "https://500px.com/USER" def __init__(self, match): _500pxExtractor.__init__(self, match) @@ -134,17 +126,7 @@ class _500pxGalleryExtractor(_500pxExtractor): directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}") pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?" r"([^/?#]+)/galleries/([^/?#]+)") - test = ( - ("https://500px.com/p/fashvamp/galleries/lera", { - "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", - "count": 3, - "keyword": { - "gallery": dict, - "user": dict, - }, - }), - ("https://500px.com/fashvamp/galleries/lera"), - ) + example = "https://500px.com/USER/galleries/GALLERY" def __init__(self, match): _500pxExtractor.__init__(self, match) @@ -200,7 +182,7 @@ class _500pxFavoriteExtractor(_500pxExtractor): """Extractor for favorite 500px photos""" subcategory = "favorite" pattern = BASE_PATTERN + r"/liked/?$" - test = ("https://500px.com/liked",) + example = "https://500px.com/liked" def photos(self): variables = {"pageSize": 20} @@ -224,50 +206,7 @@ class _500pxImageExtractor(_500pxExtractor): """Extractor for individual images from 500px.com""" subcategory = "image" pattern = BASE_PATTERN + r"/photo/(\d+)" - test = ("https://500px.com/photo/222049255/queen-of-coasts", { - "url": "fbdf7df39325cae02f5688e9f92935b0e7113315", - "count": 1, - "keyword": { - "camera": "Canon EOS 600D", - "camera_info": dict, - "comments": list, - "comments_count": int, - "created_at": "2017-08-01T08:40:05+00:00", - "description": str, - "editored_by": None, - "editors_choice": False, - "extension": "jpg", - "feature": "popular", - "feature_date": "2017-08-01T09:58:28+00:00", - "focal_length": "208", - "height": 3111, - "id": 222049255, - "image_format": "jpg", - "image_url": list, - "images": list, - "iso": "100", - "lens": "EF-S55-250mm f/4-5.6 IS II", - "lens_info": dict, - "liked": None, - "location": None, - "location_details": dict, - "name": "Queen Of Coasts", - "nsfw": False, - "privacy": False, - "profile": True, - "rating": float, - "status": 1, - "tags": list, - "taken_at": "2017-05-04T17:36:51+00:00", - "times_viewed": int, - "url": "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", - "user": dict, - "user_id": 12847235, - "votes_count": int, - "watermark": True, - "width": 4637, - }, - }) + example = "https://500px.com/photo/12345/TITLE" def __init__(self, match): _500pxExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 2d043868..3ecae13d 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -57,48 +57,7 @@ class _8chanThreadExtractor(_8chanExtractor): filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}" archive_fmt = "{boardUri}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://8chan.moe/vhs/res/4.html", { - "pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", - "count": 14, - "keyword": { - "archived": False, - "autoSage": False, - "boardDescription": "Film and Cinema", - "boardMarkdown": None, - "boardName": "Movies", - "boardUri": "vhs", - "creation": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", - "cyclic": False, - "email": None, - "id": "re:^[0-9a-f]{6}$", - "locked": False, - "markdown": str, - "maxFileCount": 5, - "maxFileSize": "32.00 MB", - "maxMessageLength": 8001, - "message": str, - "mime": str, - "name": "Anonymous", - "num": int, - "originalName": str, - "path": r"re:/.media/[0-9a-f]{64}\.\w+$", - "pinned": False, - "postId": int, - "signedRole": None, - "size": int, - "threadId": 4, - "thumb": r"re:/.media/t_[0-9a-f]{64}$", - "uniquePosters": 9, - "usesCustomCss": True, - "usesCustomJs": False, - "?wsPort": 8880, - "?wssPort": 2087, - }, - }), - ("https://8chan.se/vhs/res/4.html"), - ("https://8chan.cc/vhs/res/4.html"), - ) + example = "https://8chan.moe/a/res/12345.html" def __init__(self, match): _8chanExtractor.__init__(self, match) @@ -137,15 +96,7 @@ class _8chanBoardExtractor(_8chanExtractor): """Extractor for 8chan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$" - test = ( - ("https://8chan.moe/vhs/"), - ("https://8chan.moe/vhs/2.html", { - "pattern": _8chanThreadExtractor.pattern, - "count": 23, - }), - ("https://8chan.se/vhs/"), - ("https://8chan.cc/vhs/"), - ) + example = "https://8chan.moe/a/" def __init__(self, match): _8chanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index 584c6d2e..f88a0c67 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -22,51 +22,7 @@ class _8musesAlbumExtractor(Extractor): root = "https://comics.8muses.com" pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" r"(/comics/album/[^?#]+)(\?[^#]+)?") - test = ( - ("https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", { - "url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", - "pattern": r"https://comics.8muses.com/image/fl/[\w-]+", - "keyword": { - "url" : str, - "hash" : str, - "page" : int, - "count": 6, - "album": { - "id" : 10467, - "title" : "Liar", - "path" : "Fakku Comics/mogg/Liar", - "parts" : ["Fakku Comics", "mogg", "Liar"], - "private": False, - "url" : "https://comics.8muses.com/comics" - "/album/Fakku-Comics/mogg/Liar", - "parent" : 10464, - "views" : int, - "likes" : int, - "date" : "dt:2018-07-10 00:00:00", - }, - }, - }), - ("https://www.8muses.com/comics/album/Fakku-Comics/santa", { - "count": ">= 3", - "pattern": pattern, - "keyword": { - "url" : str, - "name" : str, - "private": False, - }, - }), - # custom sorting - ("https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", { - "count": ">= 70", - "keyword": {"name": r"re:^[R-Zr-z]"}, - }), - # non-ASCII characters - (("https://comics.8muses.com/comics/album/Various-Authors/Chessire88" - "/From-Trainers-to-Pokmons"), { - "count": 2, - "keyword": {"name": "re:From Trainers to Pokémons"}, - }), - ) + example = "https://comics.8muses.com/comics/album/PATH/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 8160e480..1617414a 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,17 +18,7 @@ class AdultempireGalleryExtractor(GalleryExtractor): root = "https://www.adultempire.com" pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" r"(/(\d+)/gallery\.html)") - test = ( - ("https://www.adultempire.com/5998/gallery.html", { - "range": "1", - "keyword": "5b3266e69801db0d78c22181da23bc102886e027", - "content": "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", - }), - ("https://www.adultdvdempire.com/5683/gallery.html", { - "url": "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", - "keyword": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", - }), - ) + example = "https://www.adultempire.com/12345/gallery.html" def __init__(self, match): GalleryExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py index dbc197ee..3d1fd138 100644 --- a/gallery_dl/extractor/architizer.py +++ b/gallery_dl/extractor/architizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,25 +21,7 @@ class ArchitizerProjectExtractor(GalleryExtractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{gid}_{num}" pattern = r"(?:https?://)?architizer\.com/projects/([^/?#]+)" - test = ("https://architizer.com/projects/house-lo/", { - "pattern": r"https://architizer-prod\.imgix\.net/media/mediadata" - r"/uploads/.+\.jpg$", - "keyword": { - "count": 27, - "description": str, - "firm": "Atelier Lina Bellovicova", - "gid": "225496", - "location": "Czechia", - "num": int, - "size": "1000 sqft - 3000 sqft", - "slug": "house-lo", - "status": "Built", - "subcategory": "project", - "title": "House LO", - "type": "Residential › Private House", - "year": "2020", - }, - }) + example = "https://architizer.com/projects/NAME/" def __init__(self, match): url = "{}/projects/{}/".format(self.root, match.group(1)) @@ -80,10 +62,7 @@ class ArchitizerFirmExtractor(Extractor): subcategory = "firm" root = "https://architizer.com" pattern = r"(?:https?://)?architizer\.com/firms/([^/?#]+)" - test = ("https://architizer.com/firms/olson-kundig/", { - "pattern": ArchitizerProjectExtractor.pattern, - "count": ">= 90", - }) + example = "https://architizer.com/firms/NAME/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index 77d5fbd2..d6676946 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -178,17 +178,7 @@ class ArtstationUserExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)(?:/albums/all)?" r"|((?!www)\w+)\.artstation\.com(?:/projects)?)/?$") - test = ( - ("https://www.artstation.com/sungchoi/", { - "pattern": r"https://\w+\.artstation\.com/p/assets/images" - r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "range": "1-10", - "count": ">= 10", - }), - ("https://www.artstation.com/sungchoi/albums/all/"), - ("https://sungchoi.artstation.com/"), - ("https://sungchoi.artstation.com/projects/"), - ) + example = "https://www.artstation.com/USER" def projects(self): url = "{}/users/{}/projects.json".format(self.root, self.user) @@ -205,15 +195,7 @@ class ArtstationAlbumExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)" r"|((?!www)\w+)\.artstation\.com)/albums/(\d+)") - test = ( - ("https://www.artstation.com/huimeiye/albums/770899", { - "count": 2, - }), - ("https://www.artstation.com/huimeiye/albums/770898", { - "exception": exception.NotFoundError, - }), - ("https://huimeiye.artstation.com/albums/770899"), - ) + example = "https://www.artstation.com/USER/albums/12345" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -247,17 +229,7 @@ class ArtstationLikesExtractor(ArtstationExtractor): archive_fmt = "f_{userinfo[id]}_{asset[id]}" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)/likes/?") - test = ( - ("https://www.artstation.com/mikf/likes", { - "pattern": r"https://\w+\.artstation\.com/p/assets/images" - r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "count": 6, - }), - # no likes - ("https://www.artstation.com/sungchoi/likes", { - "count": 0, - }), - ) + example = "https://www.artstation.com/USER/likes" def projects(self): url = "{}/users/{}/likes.json".format(self.root, self.user) @@ -274,14 +246,7 @@ class ArtstationChallengeExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/contests/[^/?#]+/challenges/(\d+)" r"/?(?:\?sorting=([a-z]+))?") - test = ( - ("https://www.artstation.com/contests/thu-2017/challenges/20"), - (("https://www.artstation.com/contests/beyond-human" - "/challenges/23?sorting=winners"), { - "range": "1-30", - "count": 30, - }), - ) + example = "https://www.artstation.com/contests/NAME/challenges/12345" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -327,10 +292,7 @@ class ArtstationSearchExtractor(ArtstationExtractor): archive_fmt = "s_{search[query]}_{asset[id]}" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" r"/search/?\?([^#]+)") - test = ("https://www.artstation.com/search?query=ancient&sort_by=rank", { - "range": "1-20", - "count": 20, - }) + example = "https://www.artstation.com/search?query=QUERY" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -377,10 +339,7 @@ class ArtstationArtworkExtractor(ArtstationExtractor): archive_fmt = "A_{asset[id]}" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" r"/artwork/?\?([^#]+)") - test = ("https://www.artstation.com/artwork?sorting=latest", { - "range": "1-20", - "count": 20, - }) + example = "https://www.artstation.com/artwork?sorting=SORT" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -400,32 +359,7 @@ class ArtstationImageExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:" r"(?:\w+\.)?artstation\.com/(?:artwork|projects|search)" r"|artstn\.co/p)/(\w+)") - test = ( - ("https://www.artstation.com/artwork/LQVJr", { - "pattern": r"https?://\w+\.artstation\.com/p/assets" - r"/images/images/008/760/279/4k/.+", - "content": "7b113871465fdc09d127adfdc2767d51cf45a7e9", - # SHA1 hash without _no_cache() - # "content": "44b80f9af36d40efc5a2668cdd11d36d6793bae9", - }), - # multiple images per project - ("https://www.artstation.com/artwork/Db3dy", { - "count": 4, - }), - # embedded youtube video - ("https://www.artstation.com/artwork/g4WPK", { - "range": "2", - "options": (("external", True),), - "pattern": "ytdl:https://www.youtube.com/embed/JNFfJtwwrU0", - }), - # 404 (#3016) - ("https://www.artstation.com/artwork/3q3mXB", { - "count": 0, - }), - # alternate URL patterns - ("https://sungchoi.artstation.com/projects/LQVJr"), - ("https://artstn.co/p/LQVJr"), - ) + example = "https://www.artstation.com/artwork/abcde" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -453,10 +387,7 @@ class ArtstationFollowingExtractor(ArtstationExtractor): subcategory = "following" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)/following") - test = ("https://www.artstation.com/sungchoi/following", { - "pattern": ArtstationUserExtractor.pattern, - "count": ">= 50", - }) + example = "https://www.artstation.com/USER/following" def items(self): url = "{}/users/{}/following.json".format(self.root, self.user) diff --git a/gallery_dl/extractor/aryion.py b/gallery_dl/extractor/aryion.py index 89a8319a..576bc832 100644 --- a/gallery_dl/extractor/aryion.py +++ b/gallery_dl/extractor/aryion.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -176,16 +176,7 @@ class AryionGalleryExtractor(AryionExtractor): subcategory = "gallery" categorytransfer = True pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)" - test = ( - ("https://aryion.com/g4/gallery/jameshoward", { - "options": (("recursive", False),), - "pattern": r"https://aryion\.com/g4/data\.php\?id=\d+$", - "range": "48-52", - "count": 5, - }), - ("https://aryion.com/g4/user/jameshoward"), - ("https://aryion.com/g4/latest.php?name=jameshoward"), - ) + example = "https://aryion.com/g4/gallery/USER" def __init__(self, match): AryionExtractor.__init__(self, match) @@ -215,9 +206,7 @@ class AryionTagExtractor(AryionExtractor): directory_fmt = ("{category}", "tags", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/tags\.php\?([^#]+)" - test = ("https://aryion.com/g4/tags.php?tag=star+wars&p=19", { - "count": ">= 5", - }) + example = "https://aryion.com/g4/tags.php?tag=TAG" def _init(self): self.params = text.parse_query(self.user) @@ -235,40 +224,7 @@ class AryionPostExtractor(AryionExtractor): """Extractor for individual posts on eka's portal""" subcategory = "post" pattern = BASE_PATTERN + r"/view/(\d+)" - test = ( - ("https://aryion.com/g4/view/510079", { - "url": "f233286fa5558c07ae500f7f2d5cb0799881450e", - "keyword": { - "artist" : "jameshoward", - "user" : "jameshoward", - "filename" : "jameshoward-510079-subscribestar_150", - "extension": "jpg", - "id" : 510079, - "width" : 1665, - "height" : 1619, - "size" : 784239, - "title" : "I'm on subscribestar now too!", - "description": r"re:Doesn't hurt to have a backup, right\?", - "tags" : ["Non-Vore", "subscribestar"], - "date" : "dt:2019-02-16 19:30:34", - "path" : [], - "views" : int, - "favorites": int, - "comments" : int, - "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", - }, - }), - # x-folder (#694) - ("https://aryion.com/g4/view/588928", { - "pattern": pattern, - "count": ">= 8", - }), - # x-comic-folder (#945) - ("https://aryion.com/g4/view/537379", { - "pattern": pattern, - "count": 2, - }), - ) + example = "https://aryion.com/g4/view/12345" def posts(self): post_id, self.user = self.user, None diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py index 638fedc2..54aaac4e 100644 --- a/gallery_dl/extractor/bbc.py +++ b/gallery_dl/extractor/bbc.py @@ -23,18 +23,7 @@ class BbcGalleryExtractor(GalleryExtractor): filename_fmt = "{num:>02}.{extension}" archive_fmt = "{programme}_{num}" pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$" - test = ( - ("https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", { - "pattern": r"https://ichef\.bbci\.co\.uk" - r"/images/ic/1920xn/\w+\.jpg", - "count": 37, - "keyword": { - "programme": "p084qtzs", - "path": ["BBC One", "Doctor Who", "The Timeless Children"], - }, - }), - ("https://www.bbc.co.uk/programmes/p084qtzs"), - ) + example = "https://www.bbc.co.uk/programmes/PATH" def metadata(self, page): data = util.json_loads(text.extr( @@ -72,17 +61,7 @@ class BbcProgrammeExtractor(Extractor): subcategory = "programme" root = "https://www.bbc.co.uk" pattern = BASE_PATTERN + r"[^/?#]+/galleries)(?:/?\?page=(\d+))?" - test = ( - ("https://www.bbc.co.uk/programmes/b006q2x0/galleries", { - "pattern": BbcGalleryExtractor.pattern, - "range": "1-50", - "count": ">= 50", - }), - ("https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", { - "pattern": BbcGalleryExtractor.pattern, - "count": ">= 100", - }), - ) + example = "https://www.bbc.co.uk/programmes/ID/galleries" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index cf123d91..7a88807a 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -84,43 +84,7 @@ class BehanceGalleryExtractor(BehanceExtractor): filename_fmt = "{category}_{id}_{num:>02}.{extension}" archive_fmt = "{id}_{num}" pattern = r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)" - test = ( - ("https://www.behance.net/gallery/17386197/A-Short-Story", { - "count": 2, - "url": "ab79bd3bef8d3ae48e6ac74fd995c1dfaec1b7d2", - "keyword": { - "id": 17386197, - "name": 're:"Hi". A short story about the important things ', - "owners": ["Place Studio", "Julio César Velazquez"], - "fields": ["Animation", "Character Design", "Directing"], - "tags": list, - "module": dict, - "date": "dt:2014-06-03 15:41:51", - }, - }), - ("https://www.behance.net/gallery/21324767/Nevada-City", { - "count": 6, - "url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", - "keyword": {"owners": ["Alex Strohl"]}, - }), - # 'media_collection' modules - ("https://www.behance.net/gallery/88276087/Audi-R8-RWD", { - "count": 20, - "url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", - "pattern": r"https://mir-s3-cdn-cf\.behance\.net/project_modules" - r"/source/[0-9a-f]+.[0-9a-f]+\.jpg" - }), - # 'video' modules (#1282) - ("https://www.behance.net/gallery/101185577/COLCCI", { - "pattern": r"https://cdn-prod-ccv\.adobe\.com/\w+" - r"/rend/\w+_720\.mp4\?", - "count": 3, - }), - # mature content (#4417) - ("https://www.behance.net/gallery/177464639/Kimori", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://www.behance.net/gallery/12345/TITLE" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -210,10 +174,7 @@ class BehanceUserExtractor(BehanceExtractor): subcategory = "user" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$" - test = ("https://www.behance.net/alexstrohl", { - "count": ">= 11", - "pattern": BehanceGalleryExtractor.pattern, - }) + example = "https://www.behance.net/USER" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -223,7 +184,7 @@ class BehanceUserExtractor(BehanceExtractor): endpoint = "GetProfileProjects" variables = { "username": self.user, - "after" : "MAo=", + "after" : "MAo=", # "0" in base64 } while True: @@ -241,10 +202,7 @@ class BehanceCollectionExtractor(BehanceExtractor): subcategory = "collection" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)" - test = ("https://www.behance.net/collection/71340149/inspiration", { - "count": ">= 150", - "pattern": BehanceGalleryExtractor.pattern, - }) + example = "https://www.behance.net/collection/12345/TITLE" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -253,7 +211,7 @@ class BehanceCollectionExtractor(BehanceExtractor): def galleries(self): endpoint = "GetMoodboardItemsAndRecommendations" variables = { - "afterItem": "MAo=", + "afterItem": "MAo=", # "0" in base64 "firstItem": 40, "id" : int(self.collection_id), "shouldGetItems" : True, diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index b25af8f9..4f767e0a 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -95,59 +95,8 @@ class BloggerExtractor(Extractor): class BloggerPostExtractor(BloggerExtractor): """Extractor for a single blog post""" subcategory = "post" - pattern = BASE_PATTERN + r"(/\d{4}/\d\d/[^/?#]+\.html)" - test = ( - ("https://julianbphotography.blogspot.com/2010/12/moon-rise.html", { - "url": "9928429fb62f712eb4de80f53625eccecc614aae", - "pattern": r"https://3.bp.blogspot.com/.*/s0/Icy-Moonrise-.*.jpg", - "keyword": { - "blog": { - "date" : "dt:2010-11-21 18:19:42", - "description": "", - "id" : "5623928067739466034", - "kind" : "blogger#blog", - "locale" : dict, - "name" : "Julian Bunker Photography", - "pages" : int, - "posts" : int, - "published" : "2010-11-21T10:19:42-08:00", - "updated" : str, - "url" : "http://julianbphotography.blogspot.com/", - }, - "post": { - "author" : "Julian Bunker", - "content" : str, - "date" : "dt:2010-12-26 01:08:00", - "etag" : str, - "id" : "6955139236418998998", - "kind" : "blogger#post", - "published" : "2010-12-25T17:08:00-08:00", - "replies" : "0", - "title" : "Moon Rise", - "updated" : "2011-12-06T05:21:24-08:00", - "url" : "re:.+/2010/12/moon-rise.html$", - }, - "num": int, - "url": str, - }, - }), - ("blogger:http://www.julianbunker.com/2010/12/moon-rise.html"), - # video (#587) - (("http://cfnmscenesinmovies.blogspot.com/2011/11/" - "cfnm-scene-jenna-fischer-in-office.html"), { - "pattern": r"https://.+\.googlevideo\.com/videoplayback", - }), - # image URLs with width/height (#1061) - # ("https://aaaninja.blogspot.com/2020/08/altera-boob-press-2.html", { - # "pattern": r"https://1.bp.blogspot.com/.+/s0/altera_.+png", - # }), - # new image domain (#2204) - (("https://randomthingsthroughmyletterbox.blogspot.com/2022/01" - "/bitter-flowers-by-gunnar-staalesen-blog.html"), { - "pattern": r"https://blogger.googleusercontent.com/img/a/.+=s0$", - "count": 8, - }), - ) + pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)" + example = "https://BLOG.blogspot.com/YYYY/MM/TITLE.html" def __init__(self, match): BloggerExtractor.__init__(self, match) @@ -161,17 +110,7 @@ class BloggerBlogExtractor(BloggerExtractor): """Extractor for an entire Blogger blog""" subcategory = "blog" pattern = BASE_PATTERN + r"/?$" - test = ( - ("https://julianbphotography.blogspot.com/", { - "range": "1-25", - "count": 25, - "pattern": r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg", - }), - ("blogger:https://www.kefblog.com.ng/", { - "range": "1-25", - "count": 25, - }), - ) + example = "https://BLOG.blogspot.com/" def posts(self, blog): return self.api.blog_posts(blog["id"]) @@ -181,12 +120,7 @@ class BloggerSearchExtractor(BloggerExtractor): """Extractor for Blogger search resuls""" subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)" - test = ( - ("https://julianbphotography.blogspot.com/search?q=400mm", { - "count": "< 10", - "keyword": {"query": "400mm"}, - }), - ) + example = "https://BLOG.blogspot.com/search?q=QUERY" def __init__(self, match): BloggerExtractor.__init__(self, match) @@ -203,13 +137,7 @@ class BloggerLabelExtractor(BloggerExtractor): """Extractor for Blogger posts by label""" subcategory = "label" pattern = BASE_PATTERN + r"/search/label/([^/?#]+)" - test = ( - ("https://dmmagazine.blogspot.com/search/label/D%26D", { - "range": "1-25", - "count": 25, - "keyword": {"label": "D&D"}, - }), - ) + example = "https://BLOG.blogspot.com/search/label/LABEL" def __init__(self, match): BloggerExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 03b84288..1318d0e4 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -29,53 +29,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): category = "bunkr" root = "https://bunkrr.su" pattern = r"(?:https?://)?(?:app\.)?bunkr+\.(?:la|[sr]u|is|to)/a/([^/?#]+)" - test = ( - ("https://bunkrr.su/a/Lktg9Keq", { - "pattern": r"https://cdn\.bunkr\.ru/test-テスト-\"&>-QjgneIQv\.png", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "album_id": "Lktg9Keq", - "album_name": 'test テスト "&>', - "count": 1, - "filename": 'test-テスト-"&>-QjgneIQv', - "id": "QjgneIQv", - "name": 'test-テスト-"&>', - "num": int, - }, - }), - # mp4 (#2239) - ("https://app.bunkr.ru/a/ptRHaCn2", { - "pattern": r"https://media-files\.bunkr\.ru/_-RnHoW69L\.mp4", - "content": "80e61d1dbc5896ae7ef9a28734c747b28b320471", - }), - # cdn4 - ("https://bunkr.is/a/iXTTc1o2", { - "pattern": r"https://(cdn|media-files)4\.bunkr\.ru/", - "content": "da29aae371b7adc8c5ef8e6991b66b69823791e8", - "keyword": { - "album_id": "iXTTc1o2", - "album_name": "test2", - "album_size": "691.1 KB", - "count": 2, - "description": "072022", - "filename": "re:video-wFO9FtxG|image-sZrQUeOx", - "id": "re:wFO9FtxG|sZrQUeOx", - "name": "re:video|image", - "num": int, - }, - }), - # cdn12 .ru TLD (#4147) - ("https://bunkrr.su/a/j1G29CnD", { - "pattern": r"https://(cdn12.bunkr.ru|media-files12.bunkr.la)/\w+", - "count": 8, - }), - ("https://bunkrr.su/a/Lktg9Keq"), - ("https://bunkr.la/a/Lktg9Keq"), - ("https://bunkr.su/a/Lktg9Keq"), - ("https://bunkr.ru/a/Lktg9Keq"), - ("https://bunkr.is/a/Lktg9Keq"), - ("https://bunkr.to/a/Lktg9Keq"), - ) + example = "https://bunkrr.su/a/ID" def fetch_album(self, album_id): # album metadata diff --git a/gallery_dl/extractor/catbox.py b/gallery_dl/extractor/catbox.py index 7a21d2a4..6c81f539 100644 --- a/gallery_dl/extractor/catbox.py +++ b/gallery_dl/extractor/catbox.py @@ -21,22 +21,7 @@ class CatboxAlbumExtractor(GalleryExtractor): directory_fmt = ("{category}", "{album_name} ({album_id})") archive_fmt = "{album_id}_{filename}" pattern = r"(?:https?://)?(?:www\.)?catbox\.moe(/c/[^/?#]+)" - test = ( - ("https://catbox.moe/c/1igcbe", { - "url": "35866a88c29462814f103bc22ec031eaeb380f8a", - "content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", - "pattern": r"https://files\.catbox\.moe/\w+\.\w{3}$", - "count": 3, - "keyword": { - "album_id": "1igcbe", - "album_name": "test", - "date": "dt:2022-08-18 00:00:00", - "description": "album test &>", - }, - }), - ("https://www.catbox.moe/c/cd90s1"), - ("https://catbox.moe/c/w7tm47#"), - ) + example = "https://catbox.moe/c/ID" def metadata(self, page): extr = text.extract_from(page) @@ -62,15 +47,7 @@ class CatboxFileExtractor(Extractor): subcategory = "file" archive_fmt = "{filename}" pattern = r"(?:https?://)?(?:files|litter|de)\.catbox\.moe/([^/?#]+)" - test = ( - ("https://files.catbox.moe/8ih3y7.png", { - "pattern": r"^https://files\.catbox\.moe/8ih3y7\.png$", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "count": 1, - }), - ("https://litter.catbox.moe/t8v3n9.png"), - ("https://de.catbox.moe/bjdmz1.jpg"), - ) + example = "https://files.catbox.moe/NAME.EXT" def items(self): url = text.ensure_http_scheme(self.url) diff --git a/gallery_dl/extractor/comicvine.py b/gallery_dl/extractor/comicvine.py index 3a578864..36dc7421 100644 --- a/gallery_dl/extractor/comicvine.py +++ b/gallery_dl/extractor/comicvine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -25,19 +25,7 @@ class ComicvineTagExtractor(BooruExtractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?comicvine\.gamespot\.com" r"(/([^/?#]+)/(\d+-\d+)/images/.*)") - test = ( - ("https://comicvine.gamespot.com/jock/4040-5653/images/", { - "pattern": r"https://comicvine\.gamespot\.com/a/uploads" - r"/original/\d+/\d+/\d+-.+\.(jpe?g|png)", - "count": ">= 140", - }), - (("https://comicvine.gamespot.com/batman/4005-1699" - "/images/?tag=Fan%20Art%20%26%20Cosplay"), { - "pattern": r"https://comicvine\.gamespot\.com/a/uploads" - r"/original/\d+/\d+/\d+-.+", - "count": ">= 450", - }), - ) + example = "https://comicvine.gamespot.com/TAG/ID/images/" def __init__(self, match): BooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 6f48e49d..61861dff 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -34,8 +34,8 @@ class Extractor(): archive_fmt = "" cookies_domain = "" browser = None + example = "" root = "" - test = None request_interval = 0.0 request_interval_min = 0.0 request_timestamp = 0.0 diff --git a/gallery_dl/extractor/cyberdrop.py b/gallery_dl/extractor/cyberdrop.py index 7a79eca6..59fd1e58 100644 --- a/gallery_dl/extractor/cyberdrop.py +++ b/gallery_dl/extractor/cyberdrop.py @@ -14,32 +14,7 @@ class CyberdropAlbumExtractor(lolisafe.LolisafeAlbumExtractor): category = "cyberdrop" root = "https://cyberdrop.me" pattern = r"(?:https?://)?(?:www\.)?cyberdrop\.(?:me|to)/a/([^/?#]+)" - test = ( - # images - ("https://cyberdrop.me/a/keKRjm4t", { - "pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$", - "keyword": { - "album_id": "keKRjm4t", - "album_name": "Fate (SFW)", - "album_size": 150069254, - "count": 62, - "date": "dt:2020-06-18 13:14:20", - "description": "", - "id": r"re:\w{8}", - }, - }), - # videos - ("https://cyberdrop.to/a/l8gIAXVD", { - "pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$", - "count": 31, - "keyword": { - "album_id": "l8gIAXVD", - "album_name": "Achelois17 videos", - "album_size": 652037121, - "date": "dt:2020-06-16 15:40:44", - }, - }), - ) + example = "https://cyberdrop.me/a/ID" def fetch_album(self, album_id): url = self.root + "/a/" + self.album_id diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index b16d27a2..f69b0e25 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -173,38 +173,7 @@ class DanbooruTagExtractor(DanbooruExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=([^&#]*)" - test = ( - ("https://danbooru.donmai.us/posts?tags=bonocho", { - "content": "b196fb9f1668109d7774a0a82efea3ffdda07746", - }), - # test page transitions - ("https://danbooru.donmai.us/posts?tags=mushishi", { - "count": ">= 300", - }), - # 'external' option (#1747) - ("https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", { - "options": (("external", True),), - "pattern": r"https://i\.pximg\.net/img-original/img" - r"/2008/08/28/02/35/48/1476533_p0\.jpg", - }), - ("https://booru.allthefallen.moe/posts?tags=yume_shokunin", { - "count": 12, - }), - ("https://aibooru.online/posts?tags=center_frills&z=1", { - "pattern": r"https://cdn\.aibooru\.online/original" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "count": ">= 3", - }), - ("https://booru.borvar.art/posts?tags=chibi&z=1", { - "pattern": r"https://booru\.borvar\.art/data/original" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "count": ">= 3", - }), - ("https://hijiribe.donmai.us/posts?tags=bonocho"), - ("https://sonohara.donmai.us/posts?tags=bonocho"), - ("https://safebooru.donmai.us/posts?tags=bonocho"), - ("https://safe.aibooru.online/posts?tags=center_frills"), - ) + example = "https://danbooru.donmai.us/posts?tags=TAG" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -238,21 +207,7 @@ class DanbooruPoolExtractor(DanbooruExtractor): directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") archive_fmt = "p_{pool[id]}_{id}" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" - test = ( - ("https://danbooru.donmai.us/pools/7659", { - "content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", - }), - ("https://booru.allthefallen.moe/pools/9", { - "url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", - "count": 6, - }), - ("https://booru.borvar.art/pools/2", { - "url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", - "count": 4, - }), - ("https://aibooru.online/pools/1"), - ("https://danbooru.donmai.us/pool/show/7659"), - ) + example = "https://danbooru.donmai.us/pools/12345" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -275,26 +230,7 @@ class DanbooruPostExtractor(DanbooruExtractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" - test = ( - ("https://danbooru.donmai.us/posts/294929", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "keyword": {"date": "dt:2008-08-12 04:46:05"}, - }), - ("https://danbooru.donmai.us/posts/3613024", { - "pattern": r"https?://.+\.zip$", - "options": (("ugoira", True),) - }), - ("https://booru.allthefallen.moe/posts/22", { - "content": "21dda68e1d7e0a554078e62923f537d8e895cac8", - }), - ("https://aibooru.online/posts/1", { - "content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", - }), - ("https://booru.borvar.art/posts/1487", { - "content": "91273ac1ea413a12be468841e2b5804656a50bff", - }), - ("https://danbooru.donmai.us/post/show/294929"), - ) + example = "https://danbooru.donmai.us/posts/12345" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -315,17 +251,7 @@ class DanbooruPopularExtractor(DanbooruExtractor): directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" pattern = BASE_PATTERN + r"/(?:explore/posts/)?popular(?:\?([^#]*))?" - test = ( - ("https://danbooru.donmai.us/explore/posts/popular"), - (("https://danbooru.donmai.us/explore/posts/popular" - "?date=2013-06-06&scale=week"), { - "range": "1-120", - "count": 120, - }), - ("https://booru.allthefallen.moe/explore/posts/popular"), - ("https://aibooru.online/explore/posts/popular"), - ("https://booru.borvar.art/explore/posts/popular"), - ) + example = "https://danbooru.donmai.us/explore/posts/popular" def __init__(self, match): DanbooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/desktopography.py b/gallery_dl/extractor/desktopography.py index 363341a5..35bb2992 100644 --- a/gallery_dl/extractor/desktopography.py +++ b/gallery_dl/extractor/desktopography.py @@ -23,7 +23,7 @@ class DesktopographySiteExtractor(DesktopographyExtractor): """Extractor for all desktopography exhibitions """ subcategory = "site" pattern = BASE_PATTERN + r"/$" - test = ("https://desktopography.net/",) + example = "https://desktopography.net/" def items(self): page = self.request(self.root).text @@ -42,7 +42,7 @@ class DesktopographyExhibitionExtractor(DesktopographyExtractor): """Extractor for a yearly desktopography exhibition""" subcategory = "exhibition" pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/" - test = ("https://desktopography.net/exhibition-2020/",) + example = "https://desktopography.net/exhibition-2020/" def __init__(self, match): DesktopographyExtractor.__init__(self, match) @@ -71,7 +71,7 @@ class DesktopographyEntryExtractor(DesktopographyExtractor): """Extractor for all resolutions of a desktopography wallpaper""" subcategory = "entry" pattern = BASE_PATTERN + r"/portfolios/([\w-]+)" - test = ("https://desktopography.net/portfolios/new-era/",) + example = "https://desktopography.net/portfolios/NAME/" def __init__(self, match): DesktopographyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 1cca967e..b8e7164c 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -440,18 +440,7 @@ class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" subcategory = "user" pattern = BASE_PATTERN + r"/?$" - test = ( - ("https://www.deviantart.com/shimoda7", { - "pattern": r"/shimoda7/gallery$", - }), - ("https://www.deviantart.com/shimoda7", { - "options": (("include", "all"),), - "pattern": r"/shimoda7/" - r"(gallery(/scraps)?|posts(/statuses)?|favourites)$", - "count": 5, - }), - ("https://shimoda7.deviantart.com/"), - ) + example = "https://www.deviantart.com/USER" def initialize(self): pass @@ -475,84 +464,7 @@ class DeviantartGalleryExtractor(DeviantartExtractor): subcategory = "gallery" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/", { - "pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.(jpg|png)\?token=.+", - "count": ">= 30", - "keyword": { - "allows_comments": bool, - "author": { - "type": "regular", - "usericon": str, - "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", - "username": "shimoda7", - }, - "category_path": str, - "content": { - "filesize": int, - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "da_category": str, - "date": "type:datetime", - "deviationid": str, - "?download_filesize": int, - "extension": str, - "index": int, - "is_deleted": bool, - "is_downloadable": bool, - "is_favourited": bool, - "is_mature": bool, - "preview": { - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "published_time": int, - "stats": { - "comments": int, - "favourites": int, - }, - "target": dict, - "thumbs": list, - "title": str, - "url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", - "username": "shimoda7", - }, - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery", { - "pattern": r"https://www.deviantart.com/yakuzafc/gallery" - r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", - "count": ">= 15", - }), - # 'folders' option (#276) - ("https://www.deviantart.com/justatest235723/gallery", { - "count": 3, - "options": (("metadata", 1), ("folders", 1), ("original", 0)), - "keyword": { - "description": str, - "folders": list, - "is_watching": bool, - "license": str, - "tags": list, - }, - }), - ("https://www.deviantart.com/shimoda8/gallery/", { - "exception": exception.NotFoundError, - }), - - ("https://www.deviantart.com/shimoda7/gallery"), - ("https://www.deviantart.com/shimoda7/gallery/all"), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=/"), - ("https://shimoda7.deviantart.com/gallery/"), - ("https://shimoda7.deviantart.com/gallery/all/"), - ("https://shimoda7.deviantart.com/gallery/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/gallery/" def deviations(self): if self.flat and not self.group: @@ -567,32 +479,7 @@ class DeviantartFolderExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "{folder[title]}") archive_fmt = "F_{folder[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)/([^/?#]+)" - test = ( - # user - ("https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", { - "count": 5, - "options": (("original", False),), - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", { - "count": ">= 4", - "options": (("original", False),), - }), - # uuid - (("https://www.deviantart.com/shimoda7/gallery" - "/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc"), { - "count": 5, - "options": (("original", False),), - }), - # name starts with '_', special characters (#1451) - (("https://www.deviantart.com/justatest235723" - "/gallery/69302698/-test-b-c-d-e-f-"), { - "count": 1, - "options": (("original", False),), - }), - ("https://shimoda7.deviantart.com/gallery/722019/Miscellaneous"), - ("https://yakuzafc.deviantart.com/gallery/37412168/Crafts"), - ) + example = "https://www.deviantart.com/USER/gallery/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -621,33 +508,7 @@ class DeviantartStashExtractor(DeviantartExtractor): subcategory = "stash" archive_fmt = "{index}.{extension}" pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)" - test = ( - ("https://sta.sh/022c83odnaxc", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.png\?token=.+", - "content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", - "count": 1, - }), - # multiple stash items - ("https://sta.sh/21jf51j7pzl2", { - "options": (("original", False),), - "count": 4, - }), - # downloadable, but no "content" field (#307) - ("https://sta.sh/024t4coz16mi", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.rar\?token=.+", - "count": 1, - }), - # mixed folders and images (#659) - ("https://sta.sh/215twi387vfj", { - "options": (("original", False),), - "count": 4, - }), - ("https://sta.sh/abcdefghijkl", { - "count": 0, - }), - ) + example = "https://sta.sh/abcde" skip = Extractor.skip @@ -692,21 +553,7 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" - test = ( - # (#271) - ("https://www.deviantart.com/h3813067/favourites/", { - "options": (("metadata", True), ("flat", False)), - "count": 1, - }), - ("https://www.deviantart.com/h3813067/favourites/", { - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/h3813067/favourites/all"), - ("https://www.deviantart.com/h3813067/favourites/?catpath=/"), - ("https://h3813067.deviantart.com/favourites/"), - ("https://h3813067.deviantart.com/favourites/all"), - ("https://h3813067.deviantart.com/favourites/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/favourites/" def deviations(self): if self.flat: @@ -723,20 +570,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor): "{collection[title]}") archive_fmt = "C_{collection[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" - test = ( - (("https://www.deviantart.com/pencilshadings/favourites" - "/70595441/3D-Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - (("https://www.deviantart.com/pencilshadings/favourites" - "/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - ("https://pencilshadings.deviantart.com" - "/favourites/70595441/3D-Favorites"), - ) + example = "https://www.deviantart.com/USER/favourites/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -767,24 +601,7 @@ class DeviantartJournalExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Journal") archive_fmt = "j_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/(?:posts(?:/journals)?|journal)/?(?:\?.*)?$" - test = ( - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", - "options": (("journals", "text"),), - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "count": 0, - "options": (("journals", "none"),), - }), - ("https://www.deviantart.com/shimoda7/posts/"), - ("https://www.deviantart.com/shimoda7/journal/"), - ("https://www.deviantart.com/shimoda7/journal/?catpath=/"), - ("https://shimoda7.deviantart.com/journal/"), - ("https://shimoda7.deviantart.com/journal/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/posts/journals/" def deviations(self): return self.api.browse_user_journals(self.user, self.offset) @@ -797,45 +614,7 @@ class DeviantartStatusExtractor(DeviantartExtractor): filename_fmt = "{category}_{index}_{title}_{date}.{extension}" archive_fmt = "S_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/posts/statuses" - test = ( - ("https://www.deviantart.com/t1na/posts/statuses", { - "count": 0, - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "count": 4, - "url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074", - }), - # shared deviation - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "none"),), - "count": 1, - "pattern": r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token=", - }), - # shared sta.sh item - ("https://www.deviantart.com/vanillaghosties/posts/statuses", { - "options": (("journals", "none"), ("original", False)), - "range": "5-", - "count": 1, - "keyword": { - "index" : int, - "index_base36": "re:^[0-9a-z]+$", - "url" : "re:^https://sta.sh", - }, - }), - # "deleted" deviations in 'items' - ("https://www.deviantart.com/AndrejSKalin/posts/statuses", { - "options": (("journals", "none"), ("original", 0), - ("image-filter", "deviationid[:8] == '147C8B03'")), - "count": 2, - "archive": False, - "keyword": {"deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655"} - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "text"),), - "url": "c8744f7f733a3029116607b826321233c5ca452d", - }), - ) + example = "https://www.deviantart.com/USER/posts/statuses/" def deviations(self): for status in self.api.user_statuses(self.user, self.offset): @@ -899,19 +678,7 @@ class DeviantartPopularExtractor(DeviantartExtractor): r"(?:deviations/?)?\?order=(popular-[^/?#]+)" r"|((?:[\w-]+/)*)(popular-[^/?#]+)" r")/?(?:\?([^#]*))?") - test = ( - ("https://www.deviantart.com/?order=popular-all-time", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/popular-24-hours/?q=tree+house", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/artisan/popular-all-time/?q=tree"), - ) + example = "https://www.deviantart.com/popular-24-hours/" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -956,11 +723,7 @@ class DeviantartTagExtractor(DeviantartExtractor): directory_fmt = ("{category}", "Tags", "{search_tags}") archive_fmt = "T_{search_tags}_{index}.{extension}" pattern = r"(?:https?://)?www\.deviantart\.com/tag/([^/?#]+)" - test = ("https://www.deviantart.com/tag/nature", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }) + example = "https://www.deviantart.com/tag/TAG" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -979,10 +742,7 @@ class DeviantartWatchExtractor(DeviantartExtractor): subcategory = "watch" pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" r"/(?:watch/deviations|notifications/watch)()()") - test = ( - ("https://www.deviantart.com/watch/deviations"), - ("https://www.deviantart.com/notifications/watch"), - ) + example = "https://www.deviantart.com/watch/deviations" def deviations(self): return self.api.browse_deviantsyouwatch() @@ -992,7 +752,7 @@ class DeviantartWatchPostsExtractor(DeviantartExtractor): """Extractor for Posts from watched users""" subcategory = "watch-posts" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" - test = ("https://www.deviantart.com/watch/posts",) + example = "https://www.deviantart.com/watch/posts" def deviations(self): return self.api.browse_posts_deviantsyouwatch() @@ -1010,100 +770,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor): r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" r"(\d+)" # bare deviation ID without slug r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 - test = ( - (("https://www.deviantart.com/shimoda7/art/For-the-sake-10073852"), { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/zzz/art/zzz-1234567890", { - "exception": exception.NotFoundError, - }), - (("https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576"), { - "options": (("comments", True),), - "keyword": {"comments": list}, - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.jpg\?token=.+", - }), - # wixmp URL rewrite - (("https://www.deviantart.com/citizenfresh/art/Hverarond-789295466"), { - "pattern": (r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token="), - }), - # GIF (#242) - (("https://www.deviantart.com/skatergators/art/COM-Moni-781571783"), { - "pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-" - r"8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-" - r"f30961955bec\.gif\?token=ey...", - }), - # Flash animation with GIF preview (#1731) - ("https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.swf\?token=.+", - "keyword": { - "filename": "flash_comic_tutorial_by_yuumei-d3juatd", - "extension": "swf", - }, - }), - # sta.sh URLs from description (#302) - (("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), { - "options": (("extra", 1), ("original", 0)), - "pattern": DeviantartStashExtractor.pattern, - "range": "2-", - "count": 4, - }), - # sta.sh URL from deviation["text_content"]["body"]["features"] - (("https://www.deviantart.com" - "/cimar-wildehopps/art/Honorary-Vixen-859809305"), { - "options": (("extra", 1),), - "pattern": ("text:<!DOCTYPE html>\n|" + - DeviantartStashExtractor.pattern), - "count": 2, - }), - # journal - ("https://www.deviantart.com/shimoda7/journal/ARTility-583755752", { - "url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", - "pattern": "text:<!DOCTYPE html>\n", - }), - # journal-like post with isJournal == False (#419) - ("https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", { - "url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", - "pattern": "text:<!DOCTYPE html>\n", - }), - # /view/ URLs - ("https://deviantart.com/view/904858796/", { - "content": "8770ec40ad1c1d60f6b602b16301d124f612948f", - }), - ("http://www.deviantart.com/view/890672057", { - "content": "1497e13d925caeb13a250cd666b779a640209236", - }), - ("https://www.deviantart.com/view/706871727", { - "content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65", - }), - ("https://www.deviantart.com/view/1", { - "exception": exception.NotFoundError, - }), - # /deviation/ (#3558) - ("https://www.deviantart.com/deviation/817215762"), - # fav.me (#3558) - ("https://fav.me/ddijrpu", { - "count": 1, - }), - ("https://fav.me/dddd", { - "exception": exception.NotFoundError, - }), - # old-style URLs - ("https://shimoda7.deviantart.com" - "/art/For-the-sake-of-a-memory-10073852"), - ("https://myria-moon.deviantart.com" - "/art/Aime-Moi-part-en-vadrouille-261986576"), - ("https://zzz.deviantart.com/art/zzz-1234567890"), - # old /view/ URLs from the Wayback Machine - ("https://www.deviantart.com/view.php?id=14864502"), - ("http://www.deviantart.com/view-full.php?id=100842"), - - ("https://www.fxdeviantart.com/zzz/art/zzz-1234567890"), - ("https://www.fxdeviantart.com/view/1234567890"), - ) + example = "https://www.deviantart.com/UsER/art/TITLE-12345" skip = Extractor.skip @@ -1134,13 +801,7 @@ class DeviantartScrapsExtractor(DeviantartExtractor): archive_fmt = "s_{_username}_{index}.{extension}" cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/(?:\?catpath=)?scraps\b" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/scraps", { - "count": 12, - }), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=scraps"), - ("https://shimoda7.deviantart.com/gallery/?catpath=scraps"), - ) + example = "https://www.deviantart.com/USER/gallery/scraps" def deviations(self): self.login() @@ -1158,11 +819,7 @@ class DeviantartSearchExtractor(DeviantartExtractor): cookies_domain = ".deviantart.com" pattern = (r"(?:https?://)?www\.deviantart\.com" r"/search(?:/deviations)?/?\?([^#]+)") - test = ( - ("https://www.deviantart.com/search?q=tree"), - ("https://www.deviantart.com/search/deviations?order=popular-1-week"), - ) - + example = "https://www.deviantart.com/search?q=QUERY" skip = Extractor.skip def __init__(self, match): @@ -1213,13 +870,7 @@ class DeviantartGallerySearchExtractor(DeviantartExtractor): archive_fmt = "g_{_username}_{index}.{extension}" cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" - test = ( - ("https://www.deviantart.com/shimoda7/gallery?q=memory", { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular"), - ) + example = "https://www.deviantart.com/USER/gallery?q=QUERY" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -1251,11 +902,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor): """Extractor for user's watched users""" subcategory = "following" pattern = BASE_PATTERN + "/about#watching$" - test = ("https://www.deviantart.com/shimoda7/about#watching", { - "pattern": DeviantartUserExtractor.pattern, - "range": "1-50", - "count": 50, - }) + example = "https://www.deviantart.com/USER/about#watching" def items(self): eclipse_api = DeviantartEclipseAPI(self) diff --git a/gallery_dl/extractor/directlink.py b/gallery_dl/extractor/directlink.py index e85eb8db..26f21849 100644 --- a/gallery_dl/extractor/directlink.py +++ b/gallery_dl/extractor/directlink.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,36 +20,7 @@ class DirectlinkExtractor(Extractor): pattern = (r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." r"(?:jpe?g|jpe|png|gif|web[mp]|mp4|mkv|og[gmv]|opus))" r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$") - test = ( - (("https://en.wikipedia.org/static/images/project-logos/enwiki.png"), { - "url": "18c5d00077332e98e53be9fed2ee4be66154b88d", - "keyword": "105770a3f4393618ab7b811b731b22663b5d3794", - }), - # empty path - (("https://example.org/file.webm"), { - "url": "2d807ed7059d1b532f1bb71dc24b510b80ff943f", - "keyword": "29dad729c40fb09349f83edafa498dba1297464a", - }), - # more complex example - ("https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", { - "url": "6fb1061390f8aada3db01cb24b51797c7ee42b31", - "keyword": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", - }), - # percent-encoded characters - ("https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", { - "url": "2627e8140727fdf743f86fe18f69f99a052c9718", - "keyword": "831790fddda081bdddd14f96985ab02dc5b5341f", - }), - # upper case file extension (#296) - ("https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw" - ".JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpP" - "mZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG"), - # internationalized domain name - ("https://räksmörgås.josefsson.org/raksmorgas.jpg", { - "url": "a65667f670b194afbd1e3ea5e7a78938d36747da", - "keyword": "fd5037fe86eebd4764e176cbaf318caec0f700be", - }), - ) + example = "https://en.wikipedia.org/static/images/project-logos/enwiki.png" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index 59e8c90d..733d0d8f 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -43,18 +43,7 @@ class DynastyscansBase(): class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): """Extractor for manga-chapters from dynasty-scans.com""" pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)" - test = ( - (("http://dynasty-scans.com/chapters/" - "hitoribocchi_no_oo_seikatsu_ch33"), { - "url": "dce64e8c504118f1ab4135c00245ea12413896cb", - "keyword": "b67599703c27316a2fe4f11c3232130a1904e032", - }), - (("http://dynasty-scans.com/chapters/" - "new_game_the_spinoff_special_13"), { - "url": "dbe5bbb74da2edcfb1832895a484e2a40bc8b538", - "keyword": "6b674eb3a274999153f6be044973b195008ced2f", - }), - ) + example = "https://dynasty-scans.com/chapters/NAME" def metadata(self, page): extr = text.extract_from(page) @@ -93,10 +82,7 @@ class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): chapterclass = DynastyscansChapterExtractor reverse = False pattern = BASE_PATTERN + r"(/series/[^/?#]+)" - test = ("https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", { - "pattern": DynastyscansChapterExtractor.pattern, - "count": ">= 100", - }) + example = "https://dynasty-scans.com/series/NAME" def chapters(self, page): return [ @@ -112,16 +98,7 @@ class DynastyscansSearchExtractor(DynastyscansBase, Extractor): filename_fmt = "{image_id}.{extension}" archive_fmt = "i_{image_id}" pattern = BASE_PATTERN + r"/images/?(?:\?([^#]+))?$" - test = ( - ("https://dynasty-scans.com/images?with[]=4930&with[]=5211", { - "url": "22cf0fb64e12b29e79b0a3d26666086a48f9916a", - "keyword": "11cbc555a15528d25567977b8808e10369c4c3ee", - }), - ("https://dynasty-scans.com/images", { - "range": "1", - "count": 1, - }), - ) + example = "https://dynasty-scans.com/images?QUERY" def __init__(self, match): Extractor.__init__(self, match) @@ -150,10 +127,7 @@ class DynastyscansImageExtractor(DynastyscansSearchExtractor): """Extractor for individual images on dynasty-scans.com""" subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" - test = ("https://dynasty-scans.com/images/1245", { - "url": "15e54bd94148a07ed037f387d046c27befa043b2", - "keyword": "0d8976c2d6fbc9ed6aa712642631b96e456dc37f", - }) + example = "https://dynasty-scans.com/images/12345" def images(self): return (self.query,) diff --git a/gallery_dl/extractor/e621.py b/gallery_dl/extractor/e621.py index cb1aea40..af963bca 100644 --- a/gallery_dl/extractor/e621.py +++ b/gallery_dl/extractor/e621.py @@ -84,48 +84,13 @@ BASE_PATTERN = E621Extractor.update({ class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor): """Extractor for e621 posts from tag searches""" pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)" - test = ( - ("https://e621.net/posts?tags=anry", { - "url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", - "content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", - }), - ("https://e621.net/post/index/1/anry"), - ("https://e621.net/post?tags=anry"), - - ("https://e926.net/posts?tags=anry", { - "url": "12198b275c62ffe2de67cca676c8e64de80c425d", - "content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", - }), - ("https://e926.net/post/index/1/anry"), - ("https://e926.net/post?tags=anry"), - - ("https://e6ai.net/posts?tags=anry"), - ("https://e6ai.net/post/index/1/anry"), - ("https://e6ai.net/post?tags=anry"), - ) + example = "https://e621.net/posts?tags=TAG" class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): """Extractor for e621 pools""" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" - test = ( - ("https://e621.net/pools/73", { - "url": "1bd09a72715286a79eea3b7f09f51b3493eb579a", - "content": "91abe5d5334425d9787811d7f06d34c77974cd22", - }), - ("https://e621.net/pool/show/73"), - - ("https://e926.net/pools/73", { - "url": "6936f1b6a18c5c25bee7cad700088dbc2503481b", - "content": "91abe5d5334425d9787811d7f06d34c77974cd22", - }), - ("https://e926.net/pool/show/73"), - - ("https://e6ai.net/pools/3", { - "url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", - }), - ("https://e6ai.net/pool/show/3"), - ) + example = "https://e621.net/pools/12345" def posts(self): self.log.info("Fetching posts of pool %s", self.pool_id) @@ -151,67 +116,7 @@ class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): """Extractor for single e621 posts""" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" - test = ( - ("https://e621.net/posts/535", { - "url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", - "content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - "keyword": {"date": "dt:2007-02-17 19:02:32"}, - }), - ("https://e621.net/posts/3181052", { - "options": (("metadata", "notes,pools"),), - "pattern": r"https://static\d\.e621\.net/data/c6/8c" - r"/c68cca0643890b615f75fb2719589bff\.png", - "keyword": { - "notes": [ - { - "body": "Little Legends 2", - "created_at": "2022-05-16T13:58:38.877-04:00", - "creator_id": 517450, - "creator_name": "EeveeCuddler69", - "height": 475, - "id": 321296, - "is_active": True, - "post_id": 3181052, - "updated_at": "2022-05-16T13:59:02.050-04:00", - "version": 3, - "width": 809, - "x": 83, - "y": 117, - }, - ], - "pools": [ - { - "category": "series", - "created_at": "2022-02-17T00:29:22.669-05:00", - "creator_id": 1077440, - "creator_name": "Yeetus90", - "description": "* \"Little Legends\":/pools/27971\r\n" - "* Little Legends 2\r\n" - "* \"Little Legends 3\":/pools/27481", - "id": 27492, - "is_active": False, - "name": "Little Legends 2", - "post_count": 39, - "post_ids": list, - "updated_at": "2022-03-27T06:30:03.382-04:00" - }, - ], - }, - }), - ("https://e621.net/post/show/535"), - - ("https://e926.net/posts/535", { - "url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", - "content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - }), - ("https://e926.net/post/show/535"), - - ("https://e6ai.net/posts/23", { - "url": "3c85a806b3d9eec861948af421fe0e8ad6b8f881", - "content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", - }), - ("https://e6ai.net/post/show/23"), - ) + example = "https://e621.net/posts/12345" def posts(self): url = "{}/posts/{}.json".format(self.root, self.post_id) @@ -221,23 +126,7 @@ class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor): """Extractor for popular images from e621""" pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" - test = ( - ("https://e621.net/explore/posts/popular"), - (("https://e621.net/explore/posts/popular" - "?date=2019-06-01&scale=month"), { - "pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", - "count": ">= 70", - }), - - ("https://e926.net/explore/posts/popular"), - (("https://e926.net/explore/posts/popular" - "?date=2019-06-01&scale=month"), { - "pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", - "count": ">= 70", - }), - - ("https://e6ai.net/explore/posts/popular"), - ) + example = "https://e621.net/explore/posts/popular" def posts(self): return self._pagination("/popular.json", self.params) @@ -249,21 +138,7 @@ class E621FavoriteExtractor(E621Extractor): directory_fmt = ("{category}", "Favorites", "{user_id}") archive_fmt = "f_{user_id}_{id}" pattern = BASE_PATTERN + r"/favorites(?:\?([^#]*))?" - test = ( - ("https://e621.net/favorites"), - ("https://e621.net/favorites?page=2&user_id=53275", { - "pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", - "count": "> 260", - }), - - ("https://e926.net/favorites"), - ("https://e926.net/favorites?page=2&user_id=53275", { - "pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", - "count": "> 260", - }), - - ("https://e6ai.net/favorites"), - ) + example = "https://e621.net/favorites" def __init__(self, match): E621Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/erome.py b/gallery_dl/extractor/erome.py index cb527410..2aed678f 100644 --- a/gallery_dl/extractor/erome.py +++ b/gallery_dl/extractor/erome.py @@ -91,29 +91,7 @@ class EromeAlbumExtractor(EromeExtractor): """Extractor for albums on erome.com""" subcategory = "album" pattern = BASE_PATTERN + r"/a/(\w+)" - test = ( - ("https://www.erome.com/a/NQgdlWvk", { - "pattern": r"https://v\d+\.erome\.com/\d+" - r"/NQgdlWvk/j7jlzmYB_480p\.mp4", - "count": 1, - "keyword": { - "album_id": "NQgdlWvk", - "num": 1, - "title": "porn", - "user": "yYgWBZw8o8qsMzM", - }, - }), - ("https://www.erome.com/a/TdbZ4ogi", { - "pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", - "count": 6, - "keyword": { - "album_id": "TdbZ4ogi", - "num": int, - "title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", - "user": "yYgWBZw8o8qsMzM", - }, - }), - ) + example = "https://www.erome.com/a/ID" def albums(self): return (self.item,) @@ -122,10 +100,7 @@ class EromeAlbumExtractor(EromeExtractor): class EromeUserExtractor(EromeExtractor): subcategory = "user" pattern = BASE_PATTERN + r"/(?!a/|search\?)([^/?#]+)" - test = ("https://www.erome.com/yYgWBZw8o8qsMzM", { - "range": "1-25", - "count": 25, - }) + example = "https://www.erome.com/USER" def albums(self): url = "{}/{}".format(self.root, self.item) @@ -135,10 +110,7 @@ class EromeUserExtractor(EromeExtractor): class EromeSearchExtractor(EromeExtractor): subcategory = "search" pattern = BASE_PATTERN + r"/search\?q=([^&#]+)" - test = ("https://www.erome.com/search?q=cute", { - "range": "1-25", - "count": 25, - }) + example = "https://www.erome.com/search?q=QUERY" def albums(self): url = self.root + "/search" diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 0a60ed82..e9131136 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -109,61 +109,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): pattern = (BASE_PATTERN + r"(?:/g/(\d+)/([\da-f]{10})" r"|/s/([\da-f]{10})/(\d+)-(\d+))") - test = ( - ("https://exhentai.org/g/1200119/d55c44d3d0/", { - "options": (("original", False),), - "keyword": { - "cost": int, - "date": "dt:2018-03-18 20:14:00", - "eh_category": "Non-H", - "expunged": False, - "favorites": r"re:^[12]\d$", - "filecount": "4", - "filesize": 1488978, - "gid": 1200119, - "height": int, - "image_token": "re:[0-9a-f]{10}", - "lang": "ja", - "language": "Japanese", - "parent": "", - "rating": r"re:\d\.\d+", - "size": int, - "tags": [ - "parody:komi-san wa komyushou desu.", - "character:shouko komi", - "group:seventh lowlife", - "other:sample", - ], - "thumb": "https://exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f8" - "3bcb1630ab1350640-624622-736-1036-jpg_250.jpg", - "title": "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan de" - "su (Komi-san wa Komyushou desu) [Sample]", - "title_jpn": "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆" - "です。 (古見さんは、コミュ症です。) [見本]", - "token": "d55c44d3d0", - "torrentcount": "0", - "uploader": "klorpa", - "width": int, - }, - "content": ("2c68cff8a7ca540a78c36fdbf5fbae0260484f87", - "e9891a4c017ed0bb734cd1efba5cd03f594d31ff"), - }), - ("https://exhentai.org/g/960461/4f0e369d82/", { - "exception": exception.NotFoundError, - }), - ("http://exhentai.org/g/962698/7f02358e00/", { - "exception": exception.AuthorizationError, - }), - ("https://exhentai.org/s/f68367b4c8/1200119-3", { - "options": (("original", False),), - "count": 2, - }), - ("https://e-hentai.org/s/f68367b4c8/1200119-3", { - "options": (("original", False),), - "count": 2, - }), - ("https://g.e-hentai.org/g/1200119/d55c44d3d0/"), - ) + example = "https://e-hentai.org/g/12345/67890abcde/" def __init__(self, match): ExhentaiExtractor.__init__(self, match) @@ -481,22 +427,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor): """Extractor for exhentai search results""" subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))" - test = ( - ("https://e-hentai.org/?f_search=touhou"), - ("https://exhentai.org/?f_cats=767&f_search=touhou"), - ("https://exhentai.org/tag/parody:touhou+project"), - (("https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0" - "&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0" - "&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter"), { - "pattern": ExhentaiGalleryExtractor.pattern, - "range": "1-30", - "count": 30, - "keyword": { - "gallery_id": int, - "gallery_token": r"re:^[0-9a-f]{10}$" - }, - }), - ) + example = "https://e-hentai.org/?f_search=QUERY" def __init__(self, match): ExhentaiExtractor.__init__(self, match) @@ -553,14 +484,7 @@ class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor): """Extractor for favorited exhentai galleries""" subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?" - test = ( - ("https://e-hentai.org/favorites.php", { - "count": 1, - "pattern": r"https?://e-hentai\.org/g/1200119/d55c44d3d0" - }), - ("https://exhentai.org/favorites.php?favcat=1&f_search=touhou" - "&f_apply=Search+Favorites"), - ) + example = "https://e-hentai.org/favorites.php" def _init(self): self.search_url = self.root + "/favorites.php" diff --git a/gallery_dl/extractor/fallenangels.py b/gallery_dl/extractor/fallenangels.py index 0503dcfa..650a7075 100644 --- a/gallery_dl/extractor/fallenangels.py +++ b/gallery_dl/extractor/fallenangels.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -13,24 +13,11 @@ from .. import text, util class FallenangelsChapterExtractor(ChapterExtractor): - """Extractor for manga-chapters from fascans.com""" + """Extractor for manga chapters from fascans.com""" category = "fallenangels" pattern = (r"(?:https?://)?(manga|truyen)\.fascans\.com" r"/manga/([^/?#]+)/([^/?#]+)") - test = ( - ("https://manga.fascans.com/manga/chronos-ruler/20/1", { - "url": "4604a7914566cc2da0ff789aa178e2d1c8c241e3", - "keyword": "2dfcc50020e32cd207be88e2a8fac0933e36bdfb", - }), - ("http://truyen.fascans.com/manga/hungry-marie/8", { - "url": "1f923d9cb337d5e7bbf4323719881794a951c6ae", - "keyword": "2bdb7334c0e3eceb9946ffd3132df679b4a94f6a", - }), - ("http://manga.fascans.com/manga/rakudai-kishi-no-eiyuutan/19.5", { - "url": "273f6863966c83ea79ad5846a2866e08067d3f0e", - "keyword": "d1065685bfe0054c4ff2a0f20acb089de4cec253", - }), - ) + example = "https://manga.fascans.com/manga/NAME/CHAPTER/" def __init__(self, match): self.version, self.manga, self.chapter = match.groups() @@ -66,16 +53,7 @@ class FallenangelsMangaExtractor(MangaExtractor): chapterclass = FallenangelsChapterExtractor category = "fallenangels" pattern = r"(?:https?://)?((manga|truyen)\.fascans\.com/manga/[^/]+)/?$" - test = ( - ("https://manga.fascans.com/manga/chronos-ruler", { - "url": "eea07dd50f5bc4903aa09e2cc3e45c7241c9a9c2", - "keyword": "c414249525d4c74ad83498b3c59a813557e59d7e", - }), - ("https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", { - "url": "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b", - "keyword": "2d2a2a5d9ea5925eb9a47bb13d848967f3af086c", - }), - ) + example = "https://manga.fascans.com/manga/NAME" def __init__(self, match): url = "https://" + match.group(1) diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py index 921ddb62..4572bea6 100644 --- a/gallery_dl/extractor/fanbox.py +++ b/gallery_dl/extractor/fanbox.py @@ -10,7 +10,6 @@ from .common import Extractor, Message from .. import text import re - BASE_PATTERN = ( r"(?:https?://)?(?:" r"(?!www\.)([\w-]+)\.fanbox\.cc|" @@ -30,12 +29,12 @@ class FanboxExtractor(Extractor): def _init(self): self.embeds = self.config("embeds", True) - def items(self): if self._warning: if not self.cookies_check(("FANBOXSESSID",)): self.log.warning("no 'FANBOXSESSID' cookie set") FanboxExtractor._warning = False + def items(self): for content_body, post in self.posts(): yield Message.Directory, post yield from self._get_urls_from_post(content_body, post) @@ -243,20 +242,7 @@ class FanboxCreatorExtractor(FanboxExtractor): """Extractor for a Fanbox creator's works""" subcategory = "creator" pattern = BASE_PATTERN + r"(?:/posts)?/?$" - test = ( - ("https://xub.fanbox.cc", { - "range": "1-15", - "count": ">= 15", - "keyword": { - "creatorId" : "xub", - "tags" : list, - "title" : str, - }, - }), - ("https://xub.fanbox.cc/posts"), - ("https://www.fanbox.cc/@xub/"), - ("https://www.fanbox.cc/@xub/posts"), - ) + example = "https://USER.fanbox.cc/" def __init__(self, match): FanboxExtractor.__init__(self, match) @@ -271,55 +257,7 @@ class FanboxPostExtractor(FanboxExtractor): """Extractor for media from a single Fanbox post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.fanbox.cc/@xub/posts/1910054", { - "count": 3, - "keyword": { - "title": "えま★おうがすと", - "tags": list, - "hasAdultContent": True, - "isCoverImage": False - }, - }), - # entry post type, image embedded in html of the post - ("https://nekoworks.fanbox.cc/posts/915", { - "count": 2, - "keyword": { - "title": "【SAYORI FAN CLUB】お届け内容", - "tags": list, - "html": str, - "hasAdultContent": True - }, - }), - # article post type, imageMap, 2 twitter embeds, fanbox embed - ("https://steelwire.fanbox.cc/posts/285502", { - "options": (("embeds", True),), - "count": 10, - "keyword": { - "title": "イラスト+SS｜義足の炭鉱少年が義足を見せてくれるだけ 【全体公開版】", - "tags": list, - "articleBody": dict, - "hasAdultContent": True - }, - }), - # 'content' metadata (#3020) - ("https://www.fanbox.cc/@official-en/posts/4326303", { - "keyword": { - "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, " - r"September 5th, 2022, we are happy to announce " - r"the start of the FANBOX hashtag event " - r"#MySetupTour ! \nAbout the event\nTo join this " - r"event .+ \nPlease check this page for further " - r"details regarding the Privacy & Terms.\n" - r"https://fanbox.pixiv.help/.+/10184952456601\n\n\n" - r"Thank you for your continued support of FANBOX.$", - }, - }), - # imageMap file order (#2718) - ("https://mochirong.fanbox.cc/posts/3746116", { - "url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", - }), - ) + example = "https://USER.fanbox.cc/posts/12345" def __init__(self, match): FanboxExtractor.__init__(self, match) @@ -334,9 +272,7 @@ class FanboxRedirectExtractor(Extractor): category = "fanbox" subcategory = "redirect" pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)" - test = ("https://www.pixiv.net/fanbox/creator/52336352", { - "pattern": FanboxCreatorExtractor.pattern, - }) + example = "https://www.pixiv.net/fanbox/creator/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/fanleaks.py b/gallery_dl/extractor/fanleaks.py index 466bb8c9..886e8934 100644 --- a/gallery_dl/extractor/fanleaks.py +++ b/gallery_dl/extractor/fanleaks.py @@ -7,7 +7,7 @@ """Extractors for https://fanleaks.club/""" from .common import Extractor, Message -from .. import text, exception +from .. import text class FanleaksExtractor(Extractor): @@ -36,34 +36,10 @@ class FanleaksExtractor(Extractor): class FanleaksPostExtractor(FanleaksExtractor): - """Extractor for individual posts on fanleak.club""" + """Extractor for individual posts on fanleaks.club""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)" - test = ( - ("https://fanleaks.club/selti/880", { - "pattern": (r"https://fanleaks\.club//models" - r"/selti/images/selti_0880\.jpg"), - "keyword": { - "model_id": "selti", - "model" : "Selti", - "id" : 880, - "type" : "photo", - }, - }), - ("https://fanleaks.club/daisy-keech/1038", { - "pattern": (r"https://fanleaks\.club//models" - r"/daisy-keech/videos/daisy-keech_1038\.mp4"), - "keyword": { - "model_id": "daisy-keech", - "model" : "Daisy Keech", - "id" : 1038, - "type" : "video", - }, - }), - ("https://fanleaks.club/hannahowo/000", { - "exception": exception.NotFoundError, - }), - ) + example = "https://fanleaks.club/MODEL/12345" def __init__(self, match): FanleaksExtractor.__init__(self, match) @@ -79,22 +55,7 @@ class FanleaksModelExtractor(FanleaksExtractor): subcategory = "model" pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club" r"/(?!latest/?$)([^/?#]+)/?$") - test = ( - ("https://fanleaks.club/hannahowo", { - "pattern": (r"https://fanleaks\.club//models" - r"/hannahowo/(images|videos)/hannahowo_\d+\.\w+"), - "range" : "1-100", - "count" : 100, - }), - ("https://fanleaks.club/belle-delphine", { - "pattern": (r"https://fanleaks\.club//models" - r"/belle-delphine/(images|videos)" - r"/belle-delphine_\d+\.\w+"), - "range" : "1-100", - "count" : 100, - }), - ("https://fanleaks.club/daisy-keech"), - ) + example = "https://fanleaks.club/MODEL" def items(self): page_num = 1 @@ -102,8 +63,7 @@ class FanleaksModelExtractor(FanleaksExtractor): self.root + "/" + self.model_id, notfound="model").text data = { "model_id": self.model_id, - "model" : text.unescape( - text.extr(page, 'mt-4">', "</h1>")), + "model" : text.unescape(text.extr(page, 'mt-4">', "</h1>")), "type" : "photo", } page_url = text.extr(page, "url: '", "'") diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index ca1eeef1..b7463336 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -173,17 +173,7 @@ class FantiaCreatorExtractor(FantiaExtractor): """Extractor for a Fantia creator's works""" subcategory = "creator" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)" - test = ( - ("https://fantia.jp/fanclubs/6939", { - "range": "1-25", - "count": ">= 25", - "keyword": { - "fanclub_user_id" : 52152, - "tags" : list, - "title" : str, - }, - }), - ) + example = "https://fantia.jp/fanclubs/12345" def __init__(self, match): FantiaExtractor.__init__(self, match) @@ -198,53 +188,7 @@ class FantiaPostExtractor(FantiaExtractor): """Extractor for media from a single Fantia post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)" - test = ( - ("https://fantia.jp/posts/1166373", { - "pattern": r"https://(" - r"c\.fantia\.jp/uploads/post/file/1166373/|" - r"cc\.fantia\.jp/uploads/post_content_photo" - r"/file/732549[01]|" - r"fantia\.jp/posts/1166373/album_image\?)", - "keyword": { - "blogpost_text": r"re:^$|" - r"This is a test.\n\nThis is a test.\n\n|" - r"Link to video:\nhttps://www.youtube.com" - r"/watch\?v=5SSdvNcAagI\n\nhtml img from " - r"another site:\n\n\n\n\n\n", - "comment": "\n\n", - "content_category": "re:thumb|blog|photo_gallery", - "content_comment": str, - "content_filename": "re:|", - "content_title": r"re:Test (Blog Content \d+|Image Gallery)" - r"|thumb", - "date": "dt:2022-03-09 16:46:12", - "fanclub_id": 356320, - "fanclub_name": "Test Fantia", - "fanclub_url": "https://fantia.jp/fanclubs/356320", - "fanclub_user_id": 7487131, - "fanclub_user_name": "2022/03/08 15:13:52の名無し", - "file_url": str, - "filename": str, - "num": int, - "plan": dict, - "post_id": 1166373, - "post_title": "Test Fantia Post", - "post_url": "https://fantia.jp/posts/1166373", - "posted_at": "Thu, 10 Mar 2022 01:46:12 +0900", - "rating": "general", - "tags": [], - }, - }), - ("https://fantia.jp/posts/508363", { - "count": 6, - "keyword": { - "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", - "tags": list, - "rating": "adult", - "post_id": 508363 - }, - }), - ) + example = "https://fantia.jp/posts/12345" def __init__(self, match): FantiaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/fapachi.py b/gallery_dl/extractor/fapachi.py index ee6d15a1..6e815193 100644 --- a/gallery_dl/extractor/fapachi.py +++ b/gallery_dl/extractor/fapachi.py @@ -14,25 +14,13 @@ class FapachiPostExtractor(Extractor): """Extractor for individual posts on fapachi.com""" category = "fapachi" subcategory = "post" + root = "https://fapachi.com" directory_fmt = ("{category}", "{user}") filename_fmt = "{user}_{id}.{extension}" archive_fmt = "{user}_{id}" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search/)([^/?#]+)/media/(\d+)") - root = "https://fapachi.com" - test = ( - # NSFW - ("https://fapachi.com/sonson/media/0082", { - "pattern": (r"https://fapachi\.com/models/s/o/" - r"sonson/1/full/sonson_0082\.jpeg"), - "keyword": { - "user": "sonson", - "id" : "0082", - }, - }), - # NSFW - ("https://fapachi.com/ferxiita/media/0159"), - ) + example = "https://fapachi.com/MODEL/media/12345" def __init__(self, match): Extractor.__init__(self, match) @@ -54,17 +42,10 @@ class FapachiUserExtractor(Extractor): """Extractor for all posts from a fapachi user""" category = "fapachi" subcategory = "user" + root = "https://fapachi.com" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$") - root = "https://fapachi.com" - test = ( - ("https://fapachi.com/sonson", { - "pattern": FapachiPostExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://fapachi.com/ferxiita/page/3"), - ) + example = "https://fapachi.com/MODEL" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/fapello.py b/gallery_dl/extractor/fapello.py index d6fcb4b3..5329a17c 100644 --- a/gallery_dl/extractor/fapello.py +++ b/gallery_dl/extractor/fapello.py @@ -19,32 +19,7 @@ class FapelloPostExtractor(Extractor): archive_fmt = "{type}_{model}_{id}" pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" r"/(?!search/|popular_videos/)([^/?#]+)/(\d+)") - test = ( - ("https://fapello.com/carrykey/530/", { - "pattern": (r"https://fapello\.com/content/c/a" - r"/carrykey/1000/carrykey_0530\.jpg"), - "keyword": { - "model": "carrykey", - "id" : 530, - "type" : "photo", - "thumbnail": "", - }, - }), - ("https://fapello.com/vladislava-661/693/", { - "pattern": (r"https://cdn\.fapello\.com/content/v/l" - r"/vladislava-661/1000/vladislava-661_0693\.mp4"), - "keyword": { - "model": "vladislava-661", - "id" : 693, - "type" : "video", - "thumbnail": ("https://fapello.com/content/v/l" - "/vladislava-661/1000/vladislava-661_0693.jpg"), - }, - }), - ("https://fapello.com/carrykey/000/", { - "exception": exception.NotFoundError, - }), - ) + example = "https://fapello.com/MODEL/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -77,14 +52,7 @@ class FapelloModelExtractor(Extractor): r"/(?!top-(?:likes|followers)|popular_videos" r"|videos|trending|search/?$)" r"([^/?#]+)/?$") - test = ( - ("https://fapello.com/hyoon/", { - "pattern": FapelloPostExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://fapello.com/kobaebeefboo/"), - ) + example = "https://fapello.com/model/" def __init__(self, match): Extractor.__init__(self, match) @@ -112,22 +80,7 @@ class FapelloPathExtractor(Extractor): pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" r"|popular_videos/[^/?#]+)/?$") - test = ( - ("https://fapello.com/top-likes/", { - "pattern": FapelloModelExtractor.pattern, - "range" : "1-10", - "count" : 10, - }), - ("https://fapello.com/videos/", { - "pattern": FapelloPostExtractor.pattern, - "range" : "1-10", - "count" : 10, - }), - ("https://fapello.com/top-followers/"), - ("https://fapello.com/trending/"), - ("https://fapello.com/popular_videos/twelve_hours/"), - ("https://fapello.com/popular_videos/week/"), - ) + example = "https://fapello.com/PATH/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py index 3b18c63e..b48c035a 100644 --- a/gallery_dl/extractor/flickr.py +++ b/gallery_dl/extractor/flickr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -64,42 +64,7 @@ class FlickrImageExtractor(FlickrExtractor): r"(?:(?:www\.|secure\.|m\.)?flickr\.com/photos/[^/?#]+/" r"|[\w-]+\.static\.?flickr\.com/(?:\d+/)+)(\d+)" r"|flic\.kr/p/([A-Za-z1-9]+))") - test = ( - ("https://www.flickr.com/photos/departingyyz/16089302239", { - "pattern": pattern, - "content": ("3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", - "0821a28ee46386e85b02b67cf2720063440a228c"), - "keyword": { - "comments": int, - "description": str, - "extension": "jpg", - "filename": "16089302239_de18cd8017_b", - "id": 16089302239, - "height": 683, - "label": "Large", - "media": "photo", - "url": str, - "views": int, - "width": 1024, - }, - }), - ("https://secure.flickr.com/photos/departingyyz/16089302239"), - ("https://m.flickr.com/photos/departingyyz/16089302239"), - ("https://flickr.com/photos/departingyyz/16089302239"), - - ("https://www.flickr.com/photos/145617051@N08/46733161535", { - "count": 1, - "keyword": {"media": "video"}, - }), - ("http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", { - "pattern": pattern}), - ("https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", { - "pattern": pattern}), - ("https://flic.kr/p/FPVo9U", { - "pattern": pattern}), - ("https://www.flickr.com/photos/zzz/16089302238", { - "exception": exception.NotFoundError}), - ) + example = "https://www.flickr.com/photos/USER/12345" def __init__(self, match): FlickrExtractor.__init__(self, match) @@ -194,11 +159,7 @@ class FlickrGalleryExtractor(FlickrExtractor): "Galleries", "{gallery[gallery_id]} {gallery[title]}") archive_fmt = "g_{gallery[id]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/galleries/(\d+)" - test = (("https://www.flickr.com/photos/flickr/" - "galleries/72157681572514792/"), { - "pattern": FlickrImageExtractor.pattern, - "count": ">= 10", - }) + example = "https://www.flickr.com/photos/USER/galleries/12345/" def __init__(self, match): FlickrExtractor.__init__(self, match) @@ -219,10 +180,7 @@ class FlickrGroupExtractor(FlickrExtractor): directory_fmt = ("{category}", "Groups", "{group[groupname]}") archive_fmt = "G_{group[nsid]}_{id}" pattern = BASE_PATTERN + r"/groups/([^/?#]+)" - test = ("https://www.flickr.com/groups/bird_headshots/", { - "pattern": FlickrImageExtractor.pattern, - "count": "> 150", - }) + example = "https://www.flickr.com/groups/NAME/" def metadata(self): self.group = self.api.urls_lookupGroup(self.item_id) @@ -237,10 +195,7 @@ class FlickrUserExtractor(FlickrExtractor): subcategory = "user" archive_fmt = "u_{user[nsid]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/?$" - test = ("https://www.flickr.com/photos/shona_s/", { - "pattern": FlickrImageExtractor.pattern, - "count": 28, - }) + example = "https://www.flickr.com/photos/USER/" def photos(self): return self.api.people_getPhotos(self.user["nsid"]) @@ -252,10 +207,7 @@ class FlickrFavoriteExtractor(FlickrExtractor): directory_fmt = ("{category}", "{user[username]}", "Favorites") archive_fmt = "f_{user[nsid]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/favorites" - test = ("https://www.flickr.com/photos/shona_s/favorites", { - "pattern": FlickrImageExtractor.pattern, - "count": 4, - }) + example = "https://www.flickr.com/photos/USER/favorites" def photos(self): return self.api.favorites_getList(self.user["nsid"]) @@ -267,11 +219,7 @@ class FlickrSearchExtractor(FlickrExtractor): directory_fmt = ("{category}", "Search", "{search[text]}") archive_fmt = "s_{search}_{id}" pattern = BASE_PATTERN + r"/search/?\?([^#]+)" - test = ( - ("https://flickr.com/search/?text=mountain"), - ("https://flickr.com/search/?text=tree%20cloud%20house" - "&color_codes=4&styles=minimalism"), - ) + example = "https://flickr.com/search/?text=QUERY" def __init__(self, match): FlickrExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index fefb2c4c..b8ec9be4 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -111,43 +111,7 @@ class FoolfuukaThreadExtractor(FoolfuukaExtractor): directory_fmt = ("{category}", "{board[shortname]}", "{thread_num} {title|comment[:50]}") pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)" - test = ( - ("https://archive.4plebs.org/tg/thread/54059290", { - "url": "fd823f17b5001442b941fddcd9ec91bafedfbc79", - }), - ("https://archived.moe/gd/thread/309639/", { - "url": "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", - "content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", - }), - ("https://archived.moe/a/thread/159767162/", { - "url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", - }), - ("https://archiveofsins.com/h/thread/4668813/", { - "url": "f612d287087e10a228ef69517cf811539db9a102", - "content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", - }), - ("https://arch.b4k.co/meta/thread/196/", { - "url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", - }), - ("https://desuarchive.org/a/thread/159542679/", { - "url": "e7d624aded15a069194e38dc731ec23217a422fb", - }), - ("https://boards.fireden.net/sci/thread/11264294/", { - "url": "61cab625c95584a12a30049d054931d64f8d20aa", - }), - ("https://archive.palanq.win/c/thread/4209598/", { - "url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", - }), - ("https://rbt.asia/g/thread/61487650/", { - "url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", - }), - ("https://archive.rebeccablacktech.com/g/thread/61487650/", { - "url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", - }), - ("https://thebarchive.com/b/thread/739772332/", { - "url": "e8b18001307d130d67db31740ce57c8561b5d80c", - }), - ) + example = "https://archived.moe/a/thread/12345/" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -175,17 +139,7 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka based boards/archives""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)/\d*$" - test = ( - ("https://archive.4plebs.org/tg/"), - ("https://archived.moe/gd/"), - ("https://archiveofsins.com/h/"), - ("https://arch.b4k.co/meta/"), - ("https://desuarchive.org/a/"), - ("https://boards.fireden.net/sci/"), - ("https://archive.palanq.win/c/"), - ("https://rbt.asia/g/"), - ("https://thebarchive.com/b/"), - ) + example = "https://archived.moe/a/" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -217,18 +171,8 @@ class FoolfuukaSearchExtractor(FoolfuukaExtractor): subcategory = "search" directory_fmt = ("{category}", "search", "{search}") pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)" + example = "https://archived.moe/_/search/text/QUERY/" request_interval = 1.0 - test = ( - ("https://archive.4plebs.org/_/search/text/test/"), - ("https://archived.moe/_/search/text/test/"), - ("https://archiveofsins.com/_/search/text/test/"), - ("https://archiveofsins.com/_/search/text/test/"), - ("https://desuarchive.org/_/search/text/test/"), - ("https://boards.fireden.net/_/search/text/test/"), - ("https://archive.palanq.win/_/search/text/test/"), - ("https://rbt.asia/_/search/text/test/"), - ("https://thebarchive.com/_/search/text/test/"), - ) def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -283,17 +227,7 @@ class FoolfuukaGalleryExtractor(FoolfuukaExtractor): subcategory = "gallery" directory_fmt = ("{category}", "{board}", "gallery") pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?" - test = ( - ("https://archive.4plebs.org/tg/gallery/1"), - ("https://archived.moe/gd/gallery/2"), - ("https://archiveofsins.com/h/gallery/3"), - ("https://arch.b4k.co/meta/gallery/"), - ("https://desuarchive.org/a/gallery/5"), - ("https://boards.fireden.net/sci/gallery/6"), - ("https://archive.palanq.win/c/gallery"), - ("https://rbt.asia/g/gallery/8"), - ("https://thebarchive.com/b/gallery/9"), - ) + example = "https://archived.moe/a/gallery" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 57d37b76..6db207a1 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -53,13 +53,7 @@ class FoolslideChapterExtractor(FoolslideExtractor): "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" - test = ( - (("https://read.powermanga.org" - "/read/one_piece_digital_colour_comics/en/0/75/"), { - "url": "854c5817f8f767e1bccd05fa9d58ffb5a4b09384", - "keyword": "a60c42f2634b7387899299d411ff494ed0ad6dbe", - }), - ) + example = "https://read.powermanga.org/read/MANGA/en/0/CHAPTER/" def items(self): page = self.request(self.gallery_url).text @@ -103,23 +97,7 @@ class FoolslideMangaExtractor(FoolslideExtractor): subcategory = "manga" categorytransfer = True pattern = BASE_PATTERN + r"(/series/[^/?#]+)" - test = ( - (("https://read.powermanga.org" - "/series/one_piece_digital_colour_comics/"), { - "count": ">= 1", - "keyword": { - "chapter": int, - "chapter_minor": str, - "chapter_string": str, - "group": "PowerManga", - "lang": "en", - "language": "English", - "manga": "One Piece Digital Colour Comics", - "title": str, - "volume": int, - }, - }), - ) + example = "https://read.powermanga.org/series/MANGA/" def items(self): page = self.request(self.gallery_url).text diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 8c3ef79d..7dc64193 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -31,6 +31,7 @@ class FuraffinityExtractor(Extractor): def _init(self): self.offset = 0 + self.external = self.config("external", False) if self.config("descriptions") == "html": self._process_description = str.strip @@ -41,13 +42,12 @@ class FuraffinityExtractor(Extractor): else: self._new_layout = None - def items(self): if self._warning: if not self.cookies_check(self.cookies_names): self.log.warning("no 'a' and 'b' session cookies set") FuraffinityExtractor._warning = False - external = self.config("external", False) + def items(self): metadata = self.metadata() for post_id in util.advance(self.posts(), self.offset): post = self._parse_post(post_id) @@ -57,7 +57,7 @@ class FuraffinityExtractor(Extractor): yield Message.Directory, post yield Message.Url, post["url"], post - if external: + if self.external: for url in text.extract_iter( post["_description"], 'href="http', '"'): yield Message.Queue, "http" + url, post @@ -219,12 +219,7 @@ class FuraffinityGalleryExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's gallery""" subcategory = "gallery" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)" - test = ("https://www.furaffinity.net/gallery/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "range": "45-50", - "count": 6, - }) + example = "https://www.furaffinity.net/gallery/USER/" def posts(self): return self._pagination("gallery") @@ -235,11 +230,7 @@ class FuraffinityScrapsExtractor(FuraffinityExtractor): subcategory = "scraps" directory_fmt = ("{category}", "{user!l}", "Scraps") pattern = BASE_PATTERN + r"/scraps/([^/?#]+)" - test = ("https://www.furaffinity.net/scraps/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+(/stories)?/\d+/\d+.\w+.", - "count": ">= 3", - }) + example = "https://www.furaffinity.net/scraps/USER/" def posts(self): return self._pagination("scraps") @@ -250,13 +241,7 @@ class FuraffinityFavoriteExtractor(FuraffinityExtractor): subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "Favorites") pattern = BASE_PATTERN + r"/favorites/([^/?#]+)" - test = ("https://www.furaffinity.net/favorites/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+/\d+/\d+.\w+\.\w+", - "keyword": {"favorite_id": int}, - "range": "45-50", - "count": 6, - }) + example = "https://www.furaffinity.net/favorites/USER/" def posts(self): return self._pagination_favorites() @@ -273,19 +258,7 @@ class FuraffinitySearchExtractor(FuraffinityExtractor): subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)" - test = ( - ("https://www.furaffinity.net/search/?q=cute", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+/\d+/\d+.\w+\.\w+", - "range": "45-50", - "count": 6, - }), - # first page of search results (#2402) - ("https://www.furaffinity.net/search/?q=leaf&range=1day", { - "range": "1-3", - "count": 3, - }), - ) + example = "https://www.furaffinity.net/search/?q=QUERY" def __init__(self, match): FuraffinityExtractor.__init__(self, match) @@ -304,65 +277,7 @@ class FuraffinityPostExtractor(FuraffinityExtractor): """Extractor for individual posts on furaffinity""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)" - test = ( - ("https://www.furaffinity.net/view/21835115/", { - "pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art" - r"/mirlinthloth/music/1488278723/1480267446.mirlinthlot" - r"h_dj_fennmink_-_bude_s_4_ever\.mp3", - "keyword": { - "artist" : "mirlinthloth", - "artist_url" : "mirlinthloth", - "date" : "dt:2016-11-27 17:24:06", - "description": "A Song made playing the game Cosmic DJ.", - "extension" : "mp3", - "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", - "id" : 21835115, - "tags" : list, - "title" : "Bude's 4 Ever", - "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", - "user" : "mirlinthloth", - "views" : int, - "favorites" : int, - "comments" : int, - "rating" : "General", - "fa_category": "Music", - "theme" : "All", - "species" : "Unspecified / Any", - "gender" : "Any", - "width" : 120, - "height" : 120, - }, - }), - # 'external' option (#1492) - ("https://www.furaffinity.net/view/42166511/", { - "options": (("external", True),), - "pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/" - r"|http://www\.postybirb\.com", - "count": 2, - }), - # no tags (#2277) - ("https://www.furaffinity.net/view/45331225/", { - "keyword": { - "artist": "Kota_Remminders", - "artist_url": "kotaremminders", - "date": "dt:2022-01-03 17:49:33", - "fa_category": "Adoptables", - "filename": "1641232173.kotaremminders_chidopts1", - "gender": "Any", - "height": 905, - "id": 45331225, - "rating": "General", - "species": "Unspecified / Any", - "tags": [], - "theme": "All", - "title": "REMINDER", - "width": 1280, - }, - }), - ("https://furaffinity.net/view/21835115/"), - ("https://sfw.furaffinity.net/view/21835115/"), - ("https://www.furaffinity.net/full/21835115/"), - ) + example = "https://www.furaffinity.net/view/12345/" def posts(self): post_id = self.user @@ -375,16 +290,7 @@ class FuraffinityUserExtractor(FuraffinityExtractor): subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/user/([^/?#]+)" - test = ( - ("https://www.furaffinity.net/user/mirlinthloth/", { - "pattern": r"/gallery/mirlinthloth/$", - }), - ("https://www.furaffinity.net/user/mirlinthloth/", { - "options": (("include", "all"),), - "pattern": r"/(gallery|scraps|favorites)/mirlinthloth/$", - "count": 3, - }), - ) + example = "https://www.furaffinity.net/user/USER/" def initialize(self): pass @@ -402,11 +308,7 @@ class FuraffinityFollowingExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's watched users""" subcategory = "following" pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)" - test = ("https://www.furaffinity.net/watchlist/by/mirlinthloth/", { - "pattern": FuraffinityUserExtractor.pattern, - "range": "176-225", - "count": 50, - }) + example = "https://www.furaffinity.net/watchlist/by/USER/" def items(self): url = "{}/watchlist/by/{}/".format(self.root, self.user) diff --git a/gallery_dl/extractor/fuskator.py b/gallery_dl/extractor/fuskator.py index d6640f57..20afb5a4 100644 --- a/gallery_dl/extractor/fuskator.py +++ b/gallery_dl/extractor/fuskator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,22 +18,7 @@ class FuskatorGalleryExtractor(GalleryExtractor): category = "fuskator" root = "https://fuskator.com" pattern = r"(?:https?://)?fuskator\.com/(?:thumbs|expanded)/([^/?#]+)" - test = ( - ("https://fuskator.com/thumbs/d0GnIzXrSKU/", { - "pattern": r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", - "count": 22, - "keyword": { - "gallery_id": 473023, - "gallery_hash": "d0GnIzXrSKU", - "title": "re:Shaved Brunette Babe Maria Ryabushkina with ", - "views": int, - "score": float, - "count": 22, - "tags": list, - }, - }), - ("https://fuskator.com/expanded/gXpKzjgIidA/index.html"), - ) + example = "https://fuskator.com/thumbs/ID/" def __init__(self, match): self.gallery_hash = match.group(1) @@ -82,13 +67,7 @@ class FuskatorSearchExtractor(Extractor): subcategory = "search" root = "https://fuskator.com" pattern = r"(?:https?://)?fuskator\.com(/(?:search|page)/.+)" - test = ( - ("https://fuskator.com/search/red_swimsuit/", { - "pattern": FuskatorGalleryExtractor.pattern, - "count": ">= 40", - }), - ("https://fuskator.com/page/3/swimsuit/quality/"), - ) + example = "https://fuskator.com/search/TAG/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index e2173de9..d8528c96 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -115,18 +115,7 @@ class GelbooruTagExtractor(GelbooruBase, gelbooru_v02.GelbooruV02TagExtractor): """Extractor for images from gelbooru.com based on search-tags""" pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]+)" - test = ( - ("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", { - "count": 5, - }), - ("https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", { - "range": "196-204", - "url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", - "pattern": r"https://img\d\.gelbooru\.com" - r"/images/../../[0-9a-f]{32}\.jpg", - "count": 9, - }), - ) + example = "https://gelbooru.com/index.php?page=post&s=list&tags=TAG" class GelbooruPoolExtractor(GelbooruBase, @@ -134,11 +123,7 @@ class GelbooruPoolExtractor(GelbooruBase, """Extractor for gelbooru pools""" per_page = 45 pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)" - test = ( - ("https://gelbooru.com/index.php?page=pool&s=show&id=761", { - "count": 6, - }), - ) + example = "https://gelbooru.com/index.php?page=pool&s=show&id=12345" skip = GelbooruBase._skip_offset @@ -169,9 +154,7 @@ class GelbooruFavoriteExtractor(GelbooruBase, """Extractor for gelbooru favorites""" per_page = 100 pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" - test = ("https://gelbooru.com/index.php?page=favorites&s=view&id=279415", { - "count": 3, - }) + example = "https://gelbooru.com/index.php?page=favorites&s=view&id=12345" skip = GelbooruBase._skip_offset @@ -221,70 +204,14 @@ class GelbooruPostExtractor(GelbooruBase, r"(?=(?:[^#]+&)?page=post(?:&|#|$))" r"(?=(?:[^#]+&)?s=view(?:&|#|$))" r"(?:[^#]+&)?id=(\d+)") - test = ( - ("https://gelbooru.com/index.php?page=post&s=view&id=313638", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "count": 1, - }), - - ("https://gelbooru.com/index.php?page=post&s=view&id=313638"), - ("https://gelbooru.com/index.php?s=view&page=post&id=313638"), - ("https://gelbooru.com/index.php?page=post&id=313638&s=view"), - ("https://gelbooru.com/index.php?s=view&id=313638&page=post"), - ("https://gelbooru.com/index.php?id=313638&page=post&s=view"), - ("https://gelbooru.com/index.php?id=313638&s=view&page=post"), - - ("https://gelbooru.com/index.php?page=post&s=view&id=6018318", { - "options": (("tags", True),), - "content": "977caf22f27c72a5d07ea4d4d9719acdab810991", - "keyword": { - "tags_artist": "kirisaki_shuusei", - "tags_character": str, - "tags_copyright": "vocaloid", - "tags_general": str, - "tags_metadata": str, - }, - }), - # video - ("https://gelbooru.com/index.php?page=post&s=view&id=5938076", { - "content": "6360452fa8c2f0c1137749e81471238564df832a", - "pattern": r"https://img\d\.gelbooru\.com/images" - r"/22/61/226111273615049235b001b381707bd0\.webm", - }), - # notes - ("https://gelbooru.com/index.php?page=post&s=view&id=5997331", { - "options": (("notes", True),), - "keyword": { - "notes": [ - { - "body": "Look over this way when you talk~", - "height": 553, - "width": 246, - "x": 35, - "y": 72, - }, - { - "body": "Hey~\nAre you listening~?", - "height": 557, - "width": 246, - "x": 1233, - "y": 109, - }, - ], - }, - }), - ) + example = "https://gelbooru.com/index.php?page=post&s=view&id=12345" class GelbooruRedirectExtractor(GelbooruBase, Extractor): subcategory = "redirect" pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com" r"/redirect\.php\?s=([^&#]+)") - test = (("https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgu" - "cGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng=="), { - "pattern": r"https://gelbooru.com/index.php" - r"\?page=post&s=view&id=1830046" - }) + example = "https://gelbooru.com/redirect.php?s=BASE64" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index b6fbcb6d..9cdf904a 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -90,24 +90,7 @@ class GelbooruV01TagExtractor(GelbooruV01Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=post&s=list&tags=parody"), { - "range": "1-25", - "count": 25, - }), - (("https://illusioncards.booru.org" - "/index.php?page=post&s=list&tags=koikatsu"), { - "range": "1-25", - "count": 25, - }), - ("https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", { - "range": "1-25", - "count": 25, - }), - ("https://drawfriends.booru.org/index.php?page=post&s=list&tags=all"), - ("https://vidyart2.booru.org/index.php?page=post&s=list&tags=all"), - ) + example = "https://allgirl.booru.org/index.php?page=post&s=list&tags=TAG" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) @@ -128,21 +111,7 @@ class GelbooruV01FavoriteExtractor(GelbooruV01Extractor): archive_fmt = "f_{favorite_id}_{id}" per_page = 50 pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=favorites&s=view&id=1166"), { - "count": 2, - }), - (("https://illusioncards.booru.org" - "/index.php?page=favorites&s=view&id=84887"), { - "count": 2, - }), - ("https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", { - "count": 4, - }), - ("https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1"), - ("https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1"), - ) + example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=N" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) @@ -161,40 +130,7 @@ class GelbooruV01PostExtractor(GelbooruV01Extractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=post&s=view&id=100520"), { - "url": "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", - "content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", - }), - (("https://illusioncards.booru.org" - "/index.php?page=post&s=view&id=82746"), { - "url": "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", - "content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", - }), - ("https://allgirl.booru.org/index.php?page=post&s=view&id=107213", { - "url": "b416800d2d2b072f80d3b37cfca9cb806fb25d51", - "content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", - "keyword": { - "created_at": "2021-02-13 16:27:39", - "date": "dt:2021-02-13 16:27:39", - "file_url": "https://img.booru.org/allgirl//images/107" - "/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", - "height": "1200", - "id": "107213", - "md5": "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", - "rating": "s", - "score": str, - "source": "", - "tags": "blush dress green_eyes green_hair hatsune_miku " - "long_hair twintails vocaloid", - "uploader": "Honochi31", - "width": "1600" - }, - }), - ("https://drawfriends.booru.org/index.php?page=post&s=view&id=107474"), - ("https://vidyart2.booru.org/index.php?page=post&s=view&id=39168"), - ) + example = "https://allgirl.booru.org/index.php?page=post&s=view&id=12345" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index 28bc799e..0864b9f6 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -197,30 +197,7 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" - test = ( - ("https://rule34.xxx/index.php?page=post&s=list&tags=danraku", { - "content": ("5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", - "622e80be3f496672c44aab5c47fbc6941c61bc79"), - "pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", - "count": 2, - }), - ("https://safebooru.org/index.php?page=post&s=list&tags=bonocho", { - "url": "17c61b386530cf4c30842c9f580d15ef1cd09586", - "content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", - }), - ("https://realbooru.com/index.php?page=post&s=list&tags=wine", { - "count": ">= 64", - }), - ("https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", { - "count": ">= 120", - }), - ("https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", { - "url": "fe662b86d38c331fcac9c62af100167d404937dc", - }), - ("https://xbooru.com/index.php?page=post&s=list&tags=konoyan", { - "count": 11, - }), - ) + example = "https://safebooru.org/index.php?page=post&s=list&tags=TAG" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -239,25 +216,7 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=pool&s=show&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=pool&s=show&id=179", { - "count": 3, - }), - ("https://safebooru.org/index.php?page=pool&s=show&id=11", { - "count": 5, - }), - ("https://realbooru.com/index.php?page=pool&s=show&id=1", { - "count": 3, - }), - ("https://hypnohub.net/index.php?page=pool&s=show&id=61", { - "url": "d314826280073441a2da609f70ee814d1f4b9407", - "count": 3, - }), - ("https://xbooru.com/index.php?page=pool&s=show&id=757", { - "url": "ceeac56c179ec72301bd0b6add6355a138fdea01", - "count": 5, - }), - ) + example = "https://safebooru.org/index.php?page=pool&s=show&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -309,26 +268,7 @@ class GelbooruV02FavoriteExtractor(GelbooruV02Extractor): archive_fmt = "f_{favorite_id}_{id}" per_page = 50 pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", { - "count": 3, - }), - ("https://safebooru.org/index.php?page=favorites&s=view&id=17567", { - "count": 2, - }), - ("https://realbooru.com/index.php?page=favorites&s=view&id=274", { - "count": 2, - }), - ("https://tbib.org/index.php?page=favorites&s=view&id=7881", { - "count": 3, - }), - ("https://hypnohub.net/index.php?page=favorites&s=view&id=43546", { - "count": 3, - }), - ("https://xbooru.com/index.php?page=favorites&s=view&id=45206", { - "count": 4, - }), - ) + example = "https://safebooru.org/index.php?page=favorites&s=view&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -349,117 +289,7 @@ class GelbooruV02PostExtractor(GelbooruV02Extractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=post&s=view&id=863", { - "pattern": r"https://api-cdn\.rule34\.xxx/images" - r"/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", - "content": ("a43f418aa350039af0d11cae501396a33bbe2201", - "67b516295950867e1c1ab6bc13b35d3b762ed2a3"), - "options": (("tags", True), ("notes", True)), - "keyword": { - "tags_artist": "reverse_noise yamu_(reverse_noise)", - "tags_character": "hong_meiling", - "tags_copyright": "touhou", - "tags_general": str, - "tags_metadata": "censored translated", - "notes": [ - { - "body": "It feels angry, I'm losing myself... " - "It won't calm down!", - "height": 65, - "id": 93586, - "width": 116, - "x": 22, - "y": 333, - }, - { - "body": "REPUTATION OF RAGE", - "height": 272, - "id": 93587, - "width": 199, - "x": 78, - "y": 442, - }, - ], - - }, - }), - ("https://hypnohub.net/index.php?page=post&s=view&id=1439", { - "pattern": r"https://hypnohub\.net/images" - r"/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", - "content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", - "options": (("tags", True), ("notes", True)), - "keyword": { - "tags_artist": "brokenteapot", - "tags_character": "hsien-ko", - "tags_copyright": "capcom darkstalkers", - "tags_general": str, - "tags_metadata": "dialogue text translated", - "notes": [ - { - "body": "Master Master Master " - "Master Master Master", - "height": 83, - "id": 10577, - "width": 129, - "x": 259, - "y": 20, - }, - { - "body": "Response Response Response " - "Response Response Response", - "height": 86, - "id": 10578, - "width": 125, - "x": 126, - "y": 20, - }, - { - "body": "Obedience Obedience Obedience " - "Obedience Obedience Obedience", - "height": 80, - "id": 10579, - "width": 98, - "x": 20, - "y": 20, - }, - ], - - }, - }), - ("https://safebooru.org/index.php?page=post&s=view&id=1169132", { - "url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b", - "content": "93b293b27dabd198afafabbaf87c49863ac82f27", - "options": (("tags", True),), - "keyword": { - "tags_artist": "kawanakajima", - "tags_character": "heath_ledger ronald_mcdonald the_joker", - "tags_copyright": "dc_comics mcdonald's the_dark_knight", - "tags_general": str, - }, - }), - ("https://realbooru.com/index.php?page=post&s=view&id=668483", { - "pattern": r"https://realbooru\.com//?images/dc/b5" - r"/dcb5c0ce9ec0bf74a6930608985f4719\.jpeg", - "content": "7f5873ce3b6cd295ea2e81fcb49583098ea9c8da", - "options": (("tags", True),), - "keyword": { - "tags_general": "1girl blonde blonde_hair blue_eyes cute " - "female female_only looking_at_viewer smile " - "solo solo_female teeth", - "tags_model": "jennifer_lawrence", - }, - }), - ("https://tbib.org/index.php?page=post&s=view&id=9233957", { - "url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", - "content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", - }), - ("https://xbooru.com/index.php?page=post&s=view&id=1025649", { - "pattern": r"https://img\.xbooru\.com/images/444" - r"/f3eda549ad8b9db244ac335c7406c92f\.jpeg", - "content": "086668afd445438d491ecc11cee3ac69b4d65530", - }), - ) + example = "https://safebooru.org/index.php?page=post&s=view&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/generic.py b/gallery_dl/extractor/generic.py index 4ab26ae7..16d4340c 100644 --- a/gallery_dl/extractor/generic.py +++ b/gallery_dl/extractor/generic.py @@ -34,31 +34,7 @@ class GenericExtractor(Extractor): r"(?:\?(?P<query>[^#]*))?" # optional query r"(?:\#(?P<fragment>.*))?" # optional fragment ) - - test = ( - ("generic:https://www.nongnu.org/lzip/", { - "count": 1, - "content": "40be5c77773d3e91db6e1c5df720ee30afb62368", - "keyword": { - "description": "Lossless data compressor", - "imageurl": "https://www.nongnu.org/lzip/lzip.png", - "keywords": "lzip, clzip, plzip, lzlib, LZMA, bzip2, " - "gzip, data compression, GNU, free software", - "pageurl": "https://www.nongnu.org/lzip/", - }, - }), - # internationalized domain name - ("generic:https://räksmörgås.josefsson.org/", { - "count": 2, - "pattern": "^https://räksmörgås.josefsson.org/", - }), - ("g:https://en.wikipedia.org/Main_Page"), - ("g:https://example.org/path/to/file?que=1?&ry=2/#fragment"), - ("g:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"), - ("generic:https://en.wikipedia.org/Main_Page"), - ("generic:https://example.org/path/to/file?que=1?&ry=2/#fragment"), - ("generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"), - ) + example = "generic:https://www.nongnu.org/lzip/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gofile.py b/gallery_dl/extractor/gofile.py index 60886a9d..3928792e 100644 --- a/gallery_dl/extractor/gofile.py +++ b/gallery_dl/extractor/gofile.py @@ -4,6 +4,8 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +"""Extractors for https://gofile.io/""" + from .common import Extractor, Message from .. import text, exception from ..cache import cache, memcache @@ -17,49 +19,7 @@ class GofileFolderExtractor(Extractor): directory_fmt = ("{category}", "{name} ({code})") archive_fmt = "{id}" pattern = r"(?:https?://)?(?:www\.)?gofile\.io/d/([^/?#]+)" - test = ( - ("https://gofile.io/d/k6BomI", { - "pattern": r"https://store\d+\.gofile\.io/download" - r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}" - r"/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", - "keyword": { - "createTime": int, - "directLink": "re:https://store5.gofile.io/download/direct/.+", - "downloadCount": int, - "extension": "png", - "filename": "test-テスト-%22&!", - "folder": { - "childs": [ - "b0367d79-b8ba-407f-8342-aaf8eb815443", - "7fd4a36a-c1dd-49ff-9223-d93f7d24093f" - ], - "code": "k6BomI", - "createTime": 1654076165, - "id": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "name": "root", - "public": True, - "totalDownloadCount": int, - "totalSize": 182, - "type": "folder" - }, - "id": r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", - "link": r"re:https://store5.gofile.io/download/.+\.png", - "md5": "re:[0-9a-f]{32}", - "mimetype": "image/png", - "name": "test-テスト-%22&!.png", - "num": int, - "parentFolder": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "serverChoosen": "store5", - "size": 182, - "thumbnail": r"re:https://store5.gofile.io/download/.+\.png", - "type": "file" - }, - }), - ("https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", { - "options": (("website-token", None),), - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ) + example = "https://gofile.io/d/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py index 5b561ead..a5221409 100644 --- a/gallery_dl/extractor/hbrowse.py +++ b/gallery_dl/extractor/hbrowse.py @@ -47,11 +47,7 @@ class HbrowseChapterExtractor(HbrowseBase, ChapterExtractor): "{page:>03}.{extension}") archive_fmt = "{manga_id}_{chapter}_{page}" pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/(\d+)/c(\d+))" - test = ("https://www.hbrowse.com/10363/c00000", { - "url": "6feefbc9f4b98e20d8425ddffa9dd111791dc3e6", - "keyword": "274996f6c809e5250b6ff3abbc5147e29f89d9a5", - "content": "44578ebbe176c2c27434966aef22945787e2781e", - }) + example = "https://www.hbrowse.com/12345/c00000" def __init__(self, match): self.path, self.gid, self.chapter = match.groups() @@ -75,10 +71,7 @@ class HbrowseMangaExtractor(HbrowseBase, MangaExtractor): chapterclass = HbrowseChapterExtractor reverse = False pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/\d+)/?$" - test = ("https://www.hbrowse.com/10363", { - "url": "b89682bfb86c11d2af0dc47463804ec3ac4aadd6", - "keyword": "4b15fda1858a69de1fbf5afddfe47dd893397312", - }) + example = "https://www.hbrowse.com/12345" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hentai2read.py b/gallery_dl/extractor/hentai2read.py index e771a4fc..9ab1411f 100644 --- a/gallery_dl/extractor/hentai2read.py +++ b/gallery_dl/extractor/hentai2read.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -23,31 +23,7 @@ class Hentai2readChapterExtractor(Hentai2readBase, ChapterExtractor): """Extractor for a single manga chapter from hentai2read.com""" archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+/([^/?#]+))" - test = ( - ("https://hentai2read.com/amazon_elixir/1/", { - "url": "964b942cf492b3a129d2fe2608abfc475bc99e71", - "keyword": "85645b02d34aa11b3deb6dadd7536863476e1bad", - }), - ("https://hentai2read.com/popuni_kei_joshi_panic/2.5/", { - "pattern": r"https://hentaicdn\.com/hentai" - r"/13088/2\.5y/ccdn00\d+\.jpg", - "count": 36, - "keyword": { - "author": "Kurisu", - "chapter": 2, - "chapter_id": 75152, - "chapter_minor": ".5", - "count": 36, - "lang": "en", - "language": "English", - "manga": "Popuni Kei Joshi Panic!", - "manga_id": 13088, - "page": int, - "title": "Popuni Kei Joshi Panic! 2.5", - "type": "Original", - }, - }), - ) + example = "https://hentai2read.com/TITLE/1/" def __init__(self, match): self.chapter = match.group(2) @@ -85,31 +61,7 @@ class Hentai2readMangaExtractor(Hentai2readBase, MangaExtractor): """Extractor for hmanga from hentai2read.com""" chapterclass = Hentai2readChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+)/?$" - test = ( - ("https://hentai2read.com/amazon_elixir/", { - "url": "273073752d418ec887d7f7211e42b832e8c403ba", - "keyword": "5c1b712258e78e120907121d3987c71f834d13e1", - }), - ("https://hentai2read.com/oshikage_riot/", { - "url": "6595f920a3088a15c2819c502862d45f8eb6bea6", - "keyword": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", - }), - ("https://hentai2read.com/popuni_kei_joshi_panic/", { - "pattern": Hentai2readChapterExtractor.pattern, - "range": "2-3", - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": ".5", - "lang": "en", - "language": "English", - "manga": "Popuni Kei Joshi Panic!", - "manga_id": 13088, - "title": str, - "type": "Original", - }, - }), - ) + example = "https://hentai2read.com/TITLE/" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hentaicosplays.py b/gallery_dl/extractor/hentaicosplays.py index ac03923f..62df1925 100644 --- a/gallery_dl/extractor/hentaicosplays.py +++ b/gallery_dl/extractor/hentaicosplays.py @@ -21,36 +21,7 @@ class HentaicosplaysGalleryExtractor(GalleryExtractor): pattern = r"((?:https?://)?(?:\w{2}\.)?" \ r"(hentai-cosplays|hentai-img|porn-images-xxx)\.com)/" \ r"(?:image|story)/([\w-]+)" - test = ( - ("https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", { - "pattern": r"https://static\d?.hentai-cosplays.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 18, - "site": "hentai-cosplays", - "slug": "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", - }, - }), - ("https://fr.porn-images-xxx.com/image/enako-enako-24/", { - "pattern": r"https://static\d?.porn-images-xxx.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 11, - "site": "porn-images-xxx", - "title": str, - }, - }), - ("https://ja.hentai-img.com/image/hollow-cora-502/", { - "pattern": r"https://static\d?.hentai-img.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 2, - "site": "hentai-img", - "title": str, - }, - }), - ) + example = "https://hentai-cosplays.com/image/TITLE/" def __init__(self, match): root, self.site, self.slug = match.groups() diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py index 56ea1d4d..d4670eb9 100644 --- a/gallery_dl/extractor/hentaifoundry.py +++ b/gallery_dl/extractor/hentaifoundry.py @@ -168,7 +168,7 @@ class HentaifoundryUserExtractor(HentaifoundryExtractor): """Extractor for a hentaifoundry user profile""" subcategory = "user" pattern = BASE_PATTERN + r"/user/([^/?#]+)/profile" - test = ("https://www.hentai-foundry.com/user/Tenpura/profile",) + example = "https://www.hentai-foundry.com/user/USER/profile" def initialize(self): pass @@ -192,12 +192,7 @@ class HentaifoundryPicturesExtractor(HentaifoundryExtractor): """Extractor for all pictures of a hentaifoundry user""" subcategory = "pictures" pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)(?:/page/(\d+))?/?$" - test = ( - ("https://www.hentai-foundry.com/pictures/user/Tenpura", { - "url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", - }), - ("https://www.hentai-foundry.com/pictures/user/Tenpura/page/3"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -209,13 +204,7 @@ class HentaifoundryScrapsExtractor(HentaifoundryExtractor): subcategory = "scraps" directory_fmt = ("{category}", "{user}", "Scraps") pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)/scraps" - test = ( - ("https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", { - "url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7", - }), - ("https://www.hentai-foundry.com" - "/pictures/user/Evulchibi/scraps/page/3"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER/scraps" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -229,13 +218,7 @@ class HentaifoundryFavoriteExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "{user}", "Favorites") archive_fmt = "f_{user}_{index}" pattern = BASE_PATTERN + r"/user/([^/?#]+)/faves/pictures" - test = ( - ("https://www.hentai-foundry.com/user/Tenpura/faves/pictures", { - "url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", - }), - ("https://www.hentai-foundry.com" - "/user/Tenpura/faves/pictures/page/3"), - ) + example = "https://www.hentai-foundry.com/user/USER/faves/pictures" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -249,10 +232,7 @@ class HentaifoundryRecentExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "Recent Pictures", "{date}") archive_fmt = "r_{index}" pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)" - test = ("https://www.hentai-foundry.com/pictures/recent/2018-09-20", { - "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "range": "20-30", - }) + example = "https://www.hentai-foundry.com/pictures/recent/YYYY-MM-DD" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -268,10 +248,7 @@ class HentaifoundryPopularExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "Popular Pictures") archive_fmt = "p_{index}" pattern = BASE_PATTERN + r"/pictures/popular()" - test = ("https://www.hentai-foundry.com/pictures/popular", { - "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "range": "20-30", - }) + example = "https://www.hentai-foundry.com/pictures/popular" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -283,34 +260,8 @@ class HentaifoundryImageExtractor(HentaifoundryExtractor): subcategory = "image" pattern = (r"(https?://)?(?:www\.|pictures\.)?hentai-foundry\.com" r"/(?:pictures/user|[^/?#])/([^/?#]+)/(\d+)") - test = ( - (("https://www.hentai-foundry.com" - "/pictures/user/Tenpura/407501/shimakaze"), { - "url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3", - "content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", - "keyword": { - "artist" : "Tenpura", - "date" : "dt:2016-02-22 14:41:19", - "description": "Thank you!", - "height" : 700, - "index" : 407501, - "media" : "Other digital art", - "ratings": ["Sexual content", "Contains female nudity"], - "score" : int, - "tags" : ["collection", "kancolle", "kantai", "shimakaze"], - "title" : "shimakaze", - "user" : "Tenpura", - "views" : int, - "width" : 495, - }, - }), - ("http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", { - "pattern": "http://pictures.hentai-foundry.com/t/Tenpura/407501/", - }), - ("https://www.hentai-foundry.com/pictures/user/Tenpura/407501/"), - ("https://pictures.hentai-foundry.com" - "/t/Tenpura/407501/Tenpura-407501-shimakaze.png"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER/12345/TITLE" + skip = Extractor.skip def __init__(self, match): @@ -331,24 +282,7 @@ class HentaifoundryStoriesExtractor(HentaifoundryExtractor): subcategory = "stories" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)(?:/page/(\d+))?/?$" - test = ("https://www.hentai-foundry.com/stories/user/SnowWolf35", { - "count": ">= 35", - "keyword": { - "author" : "SnowWolf35", - "chapters" : int, - "comments" : int, - "date" : "type:datetime", - "description": str, - "index" : int, - "rating" : int, - "ratings" : list, - "status" : "re:(Inc|C)omplete", - "title" : str, - "user" : "SnowWolf35", - "views" : int, - "words" : int, - }, - }) + example = "https://www.hentai-foundry.com/stories/user/USER" def items(self): self._init_site_filters() @@ -367,11 +301,8 @@ class HentaifoundryStoryExtractor(HentaifoundryExtractor): subcategory = "story" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)" - test = (("https://www.hentai-foundry.com/stories/user/SnowWolf35" - "/26416/Overwatch-High-Chapter-Voting-Location"), { - "url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", - "keyword": {"title": "Overwatch High Chapter Voting Location"}, - }) + example = "https://www.hentai-foundry.com/stories/user/USER/12345/TITLE" + skip = Extractor.skip def __init__(self, match): diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py index a1e681d1..31a302d1 100644 --- a/gallery_dl/extractor/hentaifox.py +++ b/gallery_dl/extractor/hentaifox.py @@ -21,40 +21,7 @@ class HentaifoxBase(): class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): """Extractor for image galleries on hentaifox.com""" pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))" - test = ( - ("https://hentaifox.com/gallery/56622/", { - "pattern": r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", - "keyword": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", - "count": 24, - }), - # 'split_tag' element (#1378) - ("https://hentaifox.com/gallery/630/", { - "keyword": { - "artist": ["beti", "betty", "magi", "mimikaki"], - "characters": [ - "aerith gainsborough", - "tifa lockhart", - "yuffie kisaragi" - ], - "count": 32, - "gallery_id": 630, - "group": ["cu-little2"], - "parody": ["darkstalkers | vampire", "final fantasy vii"], - "tags": ["femdom", "fingering", "masturbation", "yuri"], - "title": "Cu-Little Bakanya～", - "type": "doujinshi", - }, - }), - # email-protected title (#4201) - ("https://hentaifox.com/gallery/35261/", { - "keyword": { - "gallery_id": 35261, - "title": "ManageM@ster!", - "artist": ["haritama hiroki"], - "group": ["studio n.ball"], - }, - }), - ) + example = "https://hentaifox.com/gallery/12345/" def __init__(self, match): GalleryExtractor.__init__(self, match) @@ -116,22 +83,7 @@ class HentaifoxSearchExtractor(HentaifoxBase, Extractor): subcategory = "search" pattern = (r"(?:https?://)?(?:www\.)?hentaifox\.com" r"(/(?:parody|tag|artist|character|search|group)/[^/?%#]+)") - test = ( - ("https://hentaifox.com/parody/touhou-project/"), - ("https://hentaifox.com/character/reimu-hakurei/"), - ("https://hentaifox.com/artist/distance/"), - ("https://hentaifox.com/search/touhou/"), - ("https://hentaifox.com/group/v-slash/"), - ("https://hentaifox.com/tag/heterochromia/", { - "pattern": HentaifoxGalleryExtractor.pattern, - "count": ">= 60", - "keyword": { - "url" : str, - "gallery_id": int, - "title" : str, - }, - }), - ) + example = "https://hentaifox.com/tag/TAG/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index 0617330b..f3f43c47 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -17,27 +17,7 @@ class HentaihandGalleryExtractor(GalleryExtractor): category = "hentaihand" root = "https://hentaihand.com" pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" - test = ( - (("https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-" - "no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-" - "no-railgun-english"), { - "pattern": r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", - "count": 50, - "keyword": { - "artists" : ["Takumi Na Muchi"], - "date" : "dt:2014-06-28 00:00:00", - "gallery_id": 37387, - "lang" : "en", - "language" : "English", - "parodies" : ["Toaru Kagaku No Railgun"], - "relationships": list, - "tags" : list, - "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", - "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", - "type" : "Doujinshi", - }, - }), - ) + example = "https://hentaihand.com/en/comic/TITLE" def __init__(self, match): self.slug = match.group(1) @@ -76,15 +56,7 @@ class HentaihandTagExtractor(Extractor): pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" r"/\w+/(parody|character|tag|artist|group|language" r"|category|relationship)/([^/?#]+)") - test = ( - ("https://hentaihand.com/en/artist/takumi-na-muchi", { - "pattern": HentaihandGalleryExtractor.pattern, - "count": ">= 6", - }), - ("https://hentaihand.com/en/tag/full-color"), - ("https://hentaihand.com/fr/language/japanese"), - ("https://hentaihand.com/zh/category/manga"), - ) + example = "https://hentaihand.com/en/tag/TAG" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py index 2297cc06..ba9558c0 100644 --- a/gallery_dl/extractor/hentaihere.py +++ b/gallery_dl/extractor/hentaihere.py @@ -23,32 +23,7 @@ class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor): """Extractor for a single manga chapter from hentaihere.com""" archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)" - test = ( - ("https://hentaihere.com/m/S13812/1/1/", { - "url": "964b942cf492b3a129d2fe2608abfc475bc99e71", - "keyword": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", - }), - ("https://hentaihere.com/m/S23048/1.5/1/", { - "pattern": r"https://hentaicdn\.com/hentai" - r"/23048/1\.5/ccdn00\d+\.jpg", - "count": 32, - "keyword": { - "author": "Shinozuka Yuuji", - "chapter": 1, - "chapter_id": 80186, - "chapter_minor": ".5", - "count": 32, - "lang": "en", - "language": "English", - "manga": "High School Slut's Love Consultation", - "manga_id": 23048, - "page": int, - "title": "High School Slut's Love Consultation + " - "Girlfriend [Full Color]", - "type": "Original", - }, - }), - ) + example = "https://hentaihere.com/m/S12345/1/1/" def __init__(self, match): self.manga_id, self.chapter = match.groups() @@ -87,26 +62,7 @@ class HentaihereMangaExtractor(HentaihereBase, MangaExtractor): """Extractor for hmanga from hentaihere.com""" chapterclass = HentaihereChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$" - test = ( - ("https://hentaihere.com/m/S13812", { - "url": "d1ba6e28bb2162e844f8559c2b2725ba0a093559", - "keyword": "5c1b712258e78e120907121d3987c71f834d13e1", - }), - ("https://hentaihere.com/m/S7608", { - "url": "6c5239758dc93f6b1b4175922836c10391b174f7", - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": "", - "lang": "en", - "language": "English", - "manga": "Oshikake Riot", - "manga_id": 7608, - "title": r"re:Oshikake Riot( \d+)?", - "type": "Original", - }, - }), - ) + example = "https://hentaihere.com/m/S12345" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index 96f7cc11..a358019e 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -72,30 +72,7 @@ class HiperdexBase(): class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor): """Extractor for manga chapters from hiperdex.com""" pattern = BASE_PATTERN + r"(/manga/([^/?#]+)/([^/?#]+))" - test = ( - ("https://hiperdex.com/manga/domestic-na-kanojo/154-5/", { - "pattern": r"https://(1st)?hiperdex\d?.(com|net|info)" - r"/wp-content/uploads/WP-manga/data" - r"/manga_\w+/[0-9a-f]{32}/\d+\.webp", - "count": 9, - "keyword": { - "artist" : "Sasuga Kei", - "author" : "Sasuga Kei", - "chapter": 154, - "chapter_minor": ".5", - "description": "re:Natsuo Fujii is in love with his teacher, ", - "genre" : list, - "manga" : "Domestic na Kanojo", - "release": 2014, - "score" : float, - "type" : "Manga", - }, - }), - ("https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex2.com/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex.net/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex.info/manga/domestic-na-kanojo/154-5/"), - ) + example = "https://hiperdex.com/manga/MANGA/CHAPTER/" def __init__(self, match): root, path, self.manga, self.chapter = match.groups() @@ -117,30 +94,7 @@ class HiperdexMangaExtractor(HiperdexBase, MangaExtractor): """Extractor for manga from hiperdex.com""" chapterclass = HiperdexChapterExtractor pattern = BASE_PATTERN + r"(/manga/([^/?#]+))/?$" - test = ( - ("https://hiperdex.com/manga/1603231576-youre-not-that-special/", { - "count": 51, - "pattern": HiperdexChapterExtractor.pattern, - "keyword": { - "artist" : "Bolp", - "author" : "Abyo4", - "chapter": int, - "chapter_minor": "", - "description": "re:I didn’t think much of the creepy girl in ", - "genre" : list, - "manga" : "You’re Not That Special!", - "release": 2019, - "score" : float, - "status" : "Completed", - "type" : "Manhwa", - }, - }), - ("https://hiperdex.com/manga/youre-not-that-special/"), - ("https://1sthiperdex.com/manga/youre-not-that-special/"), - ("https://hiperdex2.com/manga/youre-not-that-special/"), - ("https://hiperdex.net/manga/youre-not-that-special/"), - ("https://hiperdex.info/manga/youre-not-that-special/"), - ) + example = "https://hiperdex.com/manga/MANGA/" def __init__(self, match): root, path, self.manga = match.groups() @@ -176,16 +130,7 @@ class HiperdexArtistExtractor(HiperdexBase, MangaExtractor): chapterclass = HiperdexMangaExtractor reverse = False pattern = BASE_PATTERN + r"(/manga-a(?:rtist|uthor)/(?:[^/?#]+))" - test = ( - ("https://1sthiperdex.com/manga-artist/beck-ho-an/"), - ("https://hiperdex.net/manga-artist/beck-ho-an/"), - ("https://hiperdex2.com/manga-artist/beck-ho-an/"), - ("https://hiperdex.info/manga-artist/beck-ho-an/"), - ("https://hiperdex.com/manga-author/viagra/", { - "pattern": HiperdexMangaExtractor.pattern, - "count": ">= 6", - }), - ) + example = "https://hiperdex.com/manga-artist/NAME/" def __init__(self, match): self.root = text.ensure_http_scheme(match.group(1)) diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index c012c665..bc49ca31 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -23,47 +23,7 @@ class HitomiGalleryExtractor(GalleryExtractor): pattern = (r"(?:https?://)?hitomi\.la" r"/(?:manga|doujinshi|cg|gamecg|galleries|reader)" r"/(?:[^/?#]+-)?(\d+)") - test = ( - ("https://hitomi.la/galleries/867789.html", { - "pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+" - r"/[0-9a-f]{64}\.webp", - "keyword": "86af5371f38117a07407f11af689bdd460b09710", - "count": 16, - }), - # download test - ("https://hitomi.la/galleries/1401410.html", { - "range": "1", - "content": "d75d5a3d1302a48469016b20e53c26b714d17745", - }), - # Game CG with scenes (#321) - ("https://hitomi.la/galleries/733697.html", { - "count": 210, - }), - # fallback for galleries only available through /reader/ URLs - ("https://hitomi.la/galleries/1045954.html", { - "count": 1413, - }), - # gallery with "broken" redirect - ("https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", { - "count": 10, - "options": (("format", "original"),), - "pattern": r"https://[a-c]b\.hitomi\.la/images/\d+/\d+" - r"/[0-9a-f]{64}\.jpg", - }), - # no tags - ("https://hitomi.la/cg/1615823.html", { - "count": 22, - "options": (("format", "avif"),), - "pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+" - r"/[0-9a-f]{64}\.avif", - }), - ("https://hitomi.la/manga/amazon-no-hiyaku-867789.html"), - ("https://hitomi.la/manga/867789.html"), - ("https://hitomi.la/doujinshi/867789.html"), - ("https://hitomi.la/cg/867789.html"), - ("https://hitomi.la/gamecg/867789.html"), - ("https://hitomi.la/reader/867789.html"), - ) + example = "https://hitomi.la/manga/TITLE-867789.html" def __init__(self, match): self.gid = match.group(1) @@ -149,17 +109,7 @@ class HitomiTagExtractor(Extractor): pattern = (r"(?:https?://)?hitomi\.la/" r"(tag|artist|group|series|type|character)/" r"([^/?#]+)\.html") - test = ( - ("https://hitomi.la/tag/screenshots-japanese.html", { - "pattern": HitomiGalleryExtractor.pattern, - "count": ">= 35", - }), - ("https://hitomi.la/artist/a1-all-1.html"), - ("https://hitomi.la/group/initial%2Dg-all-1.html"), - ("https://hitomi.la/series/amnesia-all-1.html"), - ("https://hitomi.la/type/doujinshi-all-1.html"), - ("https://hitomi.la/character/a2-all-1.html"), - ) + example = "https://hitomi.la/tag/TAG-LANG.html" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hotleak.py b/gallery_dl/extractor/hotleak.py index 2ab9f3cc..234f0db3 100644 --- a/gallery_dl/extractor/hotleak.py +++ b/gallery_dl/extractor/hotleak.py @@ -59,30 +59,7 @@ class HotleakPostExtractor(HotleakExtractor): subcategory = "post" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/]+)/(photo|video)/(\d+)") - test = ( - ("https://hotleak.vip/kaiyakawaii/photo/1617145", { - "pattern": r"https://hotleak\.vip/storage/images/3625" - r"/1617145/fefdd5988dfcf6b98cc9e11616018868\.jpg", - "keyword": { - "id": 1617145, - "creator": "kaiyakawaii", - "type": "photo", - "filename": "fefdd5988dfcf6b98cc9e11616018868", - "extension": "jpg", - }, - }), - ("https://hotleak.vip/lilmochidoll/video/1625538", { - "pattern": r"ytdl:https://cdn8-leak\.camhdxx\.com" - r"/1661/1625538/index\.m3u8", - "keyword": { - "id": 1625538, - "creator": "lilmochidoll", - "type": "video", - "filename": "index", - "extension": "mp4", - }, - }), - ) + example = "https://hotleak.vip/MODEL/photo/12345" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -118,18 +95,7 @@ class HotleakCreatorExtractor(HotleakExtractor): subcategory = "creator" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/?#]+)/?$") - test = ( - ("https://hotleak.vip/kaiyakawaii", { - "range": "1-200", - "count": 200, - }), - ("https://hotleak.vip/stellaviolet", { - "count": "> 600" - }), - ("https://hotleak.vip/doesnotexist", { - "exception": exception.NotFoundError, - }), - ) + example = "https://hotleak.vip/MODEL" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -182,20 +148,7 @@ class HotleakCategoryExtractor(HotleakExtractor): """Extractor for hotleak categories""" subcategory = "category" pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?" - test = ( - ("https://hotleak.vip/photos", { - "pattern": HotleakPostExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://hotleak.vip/videos"), - ("https://hotleak.vip/creators", { - "pattern": HotleakCreatorExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://hotleak.vip/hot"), - ) + example = "https://hotleak.vip/photos" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -217,14 +170,7 @@ class HotleakSearchExtractor(HotleakExtractor): """Extractor for hotleak search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))" - test = ( - ("https://hotleak.vip/search?search=gallery-dl", { - "count": 0, - }), - ("https://hotleak.vip/search?search=hannah", { - "count": "> 30", - }), - ) + example = "https://hotleak.vip/search?search=QUERY" def __init__(self, match): HotleakExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py index fcac7fe2..16e40970 100644 --- a/gallery_dl/extractor/idolcomplex.py +++ b/gallery_dl/extractor/idolcomplex.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2021 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -133,20 +133,7 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/\?([^#]*)" - test = ( - ("https://idol.sankakucomplex.com/?tags=lyumos", { - "count": 5, - "range": "18-22", - "pattern": r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" - r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", - }), - ("https://idol.sankakucomplex.com/?tags=order:favcount", { - "count": 5, - "range": "18-22", - }), - ("https://idol.sankakucomplex.com" - "/?tags=lyumos+wreath&page=3&next=694215"), - ) + example = "https://idol.sankakucomplex.com/?tags=TAGS" per_page = 20 def __init__(self, match): @@ -214,9 +201,7 @@ class IdolcomplexPoolExtractor(IdolcomplexExtractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)" - test = ("https://idol.sankakucomplex.com/pool/show/145", { - "count": 3, - }) + example = "https://idol.sankakucomplex.com/pool/show/12345" per_page = 24 def __init__(self, match): @@ -251,17 +236,7 @@ class IdolcomplexPostExtractor(IdolcomplexExtractor): subcategory = "post" archive_fmt = "{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)" - test = ("https://idol.sankakucomplex.com/post/show/694215", { - "content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - "options": (("tags", True),), - "keyword": { - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_idol": str, - "tags_medium": str, - "tags_general": str, - }, - }) + example = "https://idol.sankakucomplex.com/post/show/12345" def __init__(self, match): IdolcomplexExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imagebam.py b/gallery_dl/extractor/imagebam.py index 9a3ea368..68360e96 100644 --- a/gallery_dl/extractor/imagebam.py +++ b/gallery_dl/extractor/imagebam.py @@ -9,7 +9,7 @@ """Extractors for https://www.imagebam.com/""" from .common import Extractor, Message -from .. import text, exception +from .. import text import re @@ -46,26 +46,7 @@ class ImagebamGalleryExtractor(ImagebamExtractor): archive_fmt = "{gallery_key}_{image_key}" pattern = (r"(?:https?://)?(?:www\.)?imagebam\.com" r"(/(?:gallery/|view/G)[a-zA-Z0-9]+)") - test = ( - ("https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", { - "url": "76d976788ae2757ac81694736b07b72356f5c4c8", - "keyword": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", - "content": "596e6bfa157f2c7169805d50075c2986549973a8", - }), - ("http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", { - # more than 100 images; see issue #219 - "count": 107, - "url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", - }), - ("http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", { - "exception": exception.HttpError, - }), - # /view/ path (#2378) - ("https://www.imagebam.com/view/GA3MT1", { - "url": "35018ce1e00a2d2825a33d3cd37857edaf804919", - "keyword": "3a9f98178f73694c527890c0d7ca9a92b46987ba", - }), - ) + example = "https://www.imagebam.com/view/GID" def items(self): page = self.request(self.root + self.path).text @@ -110,24 +91,7 @@ class ImagebamImageExtractor(ImagebamExtractor): archive_fmt = "{image_key}" pattern = (r"(?:https?://)?(?:\w+\.)?imagebam\.com" r"(/(?:image/|view/M|(?:[0-9a-f]{2}/){3})[a-zA-Z0-9]+)") - test = ( - ("https://www.imagebam.com/image/94d56c502511890", { - "url": "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", - "keyword": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("http://images3.imagebam.com/1d/8c/44/94d56c502511890.png"), - # NSFW (#1534) - ("https://www.imagebam.com/image/0850951366904951", { - "url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", - }), - # /view/ path (#2378) - ("https://www.imagebam.com/view/ME8JOQP", { - "url": "4dca72bbe61a0360185cf4ab2bed8265b49565b8", - "keyword": "15a494c02fd30846b41b42a26117aedde30e4ceb", - "content": "f81008666b17a42d8834c4749b910e1dc10a6e83", - }), - ) + example = "https://www.imagebam.com/view/MID" def items(self): path = self.path diff --git a/gallery_dl/extractor/imagechest.py b/gallery_dl/extractor/imagechest.py index 2babead8..9aa03329 100644 --- a/gallery_dl/extractor/imagechest.py +++ b/gallery_dl/extractor/imagechest.py @@ -18,29 +18,7 @@ class ImagechestGalleryExtractor(GalleryExtractor): category = "imagechest" root = "https://imgchest.com" pattern = r"(?:https?://)?(?:www\.)?imgchest\.com/p/([A-Za-z0-9]{11})" - test = ( - ("https://imgchest.com/p/3na7kr3by8d", { - "pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "keyword": { - "count": 3, - "gallery_id": "3na7kr3by8d", - "num": int, - "title": "Wizardry - Video Game From The Mid 80's", - }, - "url": "7328ca4ec2459378d725e3be19f661d2b045feda", - "content": "076959e65be30249a2c651fbe6090dc30ba85193", - "count": 3 - }), - # "Load More Files" button (#4028) - ("https://imgchest.com/p/9p4n3q2z7nq", { - "pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", - "count": 52, - }), - ("https://imgchest.com/p/xxxxxxxxxxx", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgchest.com/p/abcdefghijk" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py index 43ac3a35..f9726f8e 100644 --- a/gallery_dl/extractor/imagefap.py +++ b/gallery_dl/extractor/imagefap.py @@ -43,50 +43,7 @@ class ImagefapGalleryExtractor(ImagefapExtractor): """Extractor for image galleries from imagefap.com""" subcategory = "gallery" pattern = BASE_PATTERN + r"/(?:gallery\.php\?gid=|gallery/|pictures/)(\d+)" - - test = ( - ("https://www.imagefap.com/gallery/7102714", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/\d+/\d+/\d+\.jpg", - "keyword": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b", - "content": "694a0a57385980a6f90fbc296cadcd6c11ba2dab", - }), - ("https://www.imagefap.com/gallery/7876223", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/\d+/\d+/\d+\.jpg", - "keyword": { - "categories": ["Asses", "Softcore", "Pornstars"], - "count": 44, - "description": "", - "gallery_id": 7876223, - "image_id": int, - "num": int, - "tags": ["big ass", "panties", "horny", - "pussy", "exposed", "outdoor"], - "title": "Kelsi Monroe in lingerie", - "uploader": "BdRachel", - }, - "count": 44, - }), - # description (#3905) - ("https://www.imagefap.com/gallery/6180555", { - "range": "1", - "keyword": { - "categories": ["Amateur", "Softcore", "Homemade"], - "count": 36, - "description": "Nude and dressed sluts showing off the goods", - "gallery_id": 6180555, - "image_id": int, - "num": int, - "tags": [] , - "title": "Dressed or Undressed MG*", - "uploader": "splitopen", - }, - }), - ("https://www.imagefap.com/pictures/7102714"), - ("https://www.imagefap.com/gallery.php?gid=7102714"), - ("https://beta.imagefap.com/gallery.php?gid=7102714"), - ) + example = "https://www.imagefap.com/gallery/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -157,22 +114,7 @@ class ImagefapImageExtractor(ImagefapExtractor): """Extractor for single images from imagefap.com""" subcategory = "image" pattern = BASE_PATTERN + r"/photo/(\d+)" - test = ( - ("https://www.imagefap.com/photo/1962981893", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/65/196/1962981893\.jpg", - "keyword": { - "date": "21/08/2014", - "gallery_id": 7876223, - "height": 1600, - "image_id": 1962981893, - "title": "Kelsi Monroe in lingerie", - "uploader": "BdRachel", - "width": 1066, - }, - }), - ("https://beta.imagefap.com/photo/1962981893"), - ) + example = "https://www.imagefap.com/photo/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -213,35 +155,7 @@ class ImagefapFolderExtractor(ImagefapExtractor): pattern = (BASE_PATTERN + r"/(?:organizer/|" r"(?:usergallery\.php\?user(id)?=([^&#]+)&" r"|profile/([^/?#]+)/galleries\?)folderid=)(\d+|-1)") - test = ( - ("https://www.imagefap.com/organizer/409758", { - "pattern": r"https://www\.imagefap\.com/gallery/7876223", - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - "count": 1, - }), - (("https://www.imagefap.com/usergallery.php" - "?userid=1981976&folderid=409758"), { - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - }), - (("https://www.imagefap.com/usergallery.php" - "?user=BdRachel&folderid=409758"), { - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - }), - ("https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - (("https://www.imagefap.com/usergallery.php" - "?userid=1981976&folderid=-1"), { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - (("https://www.imagefap.com/usergallery.php" - "?user=BdRachel&folderid=-1"), { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - ) + example = "https://www.imagefap.com/organizer/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -293,20 +207,7 @@ class ImagefapUserExtractor(ImagefapExtractor): pattern = (BASE_PATTERN + r"/(?:profile(?:\.php\?user=|/)([^/?#]+)(?:/galleries)?" r"|usergallery\.php\?userid=(\d+))(?:$|#)") - test = ( - ("https://www.imagefap.com/profile/BdRachel", { - "pattern": ImagefapFolderExtractor.pattern, - "count": ">= 18", - }), - ("https://www.imagefap.com/usergallery.php?userid=1862791", { - "pattern": r"https://www\.imagefap\.com" - r"/profile/LucyRae/galleries\?folderid=-1", - "count": 1, - }), - ("https://www.imagefap.com/profile/BdRachel/galleries"), - ("https://www.imagefap.com/profile.php?user=BdRachel"), - ("https://beta.imagefap.com/profile.php?user=BdRachel"), - ) + example = "https://www.imagefap.com/profile/USER" def __init__(self, match): ImagefapExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 93dfa3e4..ddd4c888 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -74,37 +74,7 @@ class ImxtoImageExtractor(ImagehostImageExtractor): category = "imxto" pattern = (r"(?:https?://)?(?:www\.)?((?:imx\.to|img\.yt)" r"/(?:i/|img-)(\w+)(\.html)?)") - test = ( - # new-style URL - ("https://imx.to/i/1qdeva", { - "url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "size" : 18, - "width" : 64, - "height": 32, - "hash" : "94d56c599223c59f3feb71ea603484d1", - }, - }), - # old-style URL - ("https://imx.to/img-57a2050547b97.html", { - "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - "content": "54592f2635674c25677c6872db3709d343cdf92f", - "keyword": { - "size" : 5284, - "width" : 320, - "height": 160, - "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", - }, - }), - # img.yt domain - ("https://img.yt/img-57a2050547b97.html", { - "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - }), - ("https://imx.to/img-57a2050547b98.html", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imx.to/i/ID" _params = "simple" _encoding = "utf-8" @@ -143,11 +113,7 @@ class ImxtoGalleryExtractor(ImagehostImageExtractor): category = "imxto" subcategory = "gallery" pattern = r"(?:https?://)?(?:www\.)?(imx\.to/g/([^/?#]+))" - test = ("https://imx.to/g/ozdy", { - "pattern": ImxtoImageExtractor.pattern, - "keyword": {"title": "untitled gallery"}, - "count": 40, - }) + example = "https://imx.to/g/ID" def items(self): page = self.request(self.page_url).text @@ -165,11 +131,7 @@ class AcidimgImageExtractor(ImagehostImageExtractor): """Extractor for single images from acidimg.cc""" category = "acidimg" pattern = r"(?:https?://)?((?:www\.)?acidimg\.cc/img-([a-z0-9]+)\.html)" - test = ("https://acidimg.cc/img-5acb6b9de4640.html", { - "url": "f132a630006e8d84f52d59555191ed82b3b64c04", - "keyword": "135347ab4345002fc013863c0d9419ba32d98f78", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://acidimg.cc/img-abc123.html" _params = "simple" _encoding = "utf-8" @@ -192,22 +154,7 @@ class ImagevenueImageExtractor(ImagehostImageExtractor): category = "imagevenue" pattern = (r"(?:https?://)?((?:www|img\d+)\.imagevenue\.com" r"/([A-Z0-9]{8,10}|view/.*|img\.php\?.*))") - test = ( - ("https://www.imagevenue.com/ME13LS07", { - "pattern": r"https://cdn-images\.imagevenue\.com" - r"/10/ac/05/ME13LS07_o\.png", - "keyword": "ae15d6e3b2095f019eee84cd896700cd34b09c36", - "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", - }), - (("https://www.imagevenue.com/view/o?i=92518_13732377" - "annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486"), { - "url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", - }), - (("http://img28116.imagevenue.com/img.php" - "?image=th_52709_test_122_64lo.jpg"), { - "url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", - }), - ) + example = "https://www.imagevenue.com/ME123456789" def get_info(self, page): pos = page.index('class="card-body') @@ -223,17 +170,7 @@ class ImagetwistImageExtractor(ImagehostImageExtractor): category = "imagetwist" pattern = (r"(?:https?://)?((?:www\.|phun\.)?" r"image(?:twist|haha)\.com/([a-z0-9]{12}))") - test = ( - ("https://imagetwist.com/f1i2s4vhvbrq/test.png", { - "url": "8d5e168c0bee30211f821c6f3b2116e419d42671", - "keyword": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("https://www.imagetwist.com/f1i2s4vhvbrq/test.png"), - ("https://phun.imagetwist.com/f1i2s4vhvbrq/test.png"), - ("https://imagehaha.com/f1i2s4vhvbrq/test.png"), - ("https://www.imagehaha.com/f1i2s4vhvbrq/test.png"), - ) + example = "https://imagetwist.com/123456abcdef/NAME.EXT" @property @memcache(maxage=3*3600) @@ -250,11 +187,7 @@ class ImgspiceImageExtractor(ImagehostImageExtractor): """Extractor for single images from imgspice.com""" category = "imgspice" pattern = r"(?:https?://)?((?:www\.)?imgspice\.com/([^/?#]+))" - test = ("https://imgspice.com/nwfwtpyog50y/test.png.html", { - "url": "b8c30a8f51ee1012959a4cfd46197fabf14de984", - "keyword": "100e310a19a2fa22d87e1bbc427ecb9f6501e0c0", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://imgspice.com/ID/NAME.EXT.html" def get_info(self, page): pos = page.find('id="imgpreview"') @@ -270,11 +203,7 @@ class PixhostImageExtractor(ImagehostImageExtractor): category = "pixhost" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" r"/show/\d+/(\d+)_[^/?#]+)") - test = ("https://pixhost.to/show/190/130327671_test-.png", { - "url": "4e5470dcf6513944773044d40d883221bbc46cff", - "keyword": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://pixhost.to/show/123/12345_NAME.EXT" _cookies = {"pixhostads": "1", "pixhosttest": "1"} def get_info(self, page): @@ -289,10 +218,7 @@ class PixhostGalleryExtractor(ImagehostImageExtractor): subcategory = "gallery" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" r"/gallery/([^/?#]+))") - test = ("https://pixhost.to/gallery/jSMFq", { - "pattern": PixhostImageExtractor.pattern, - "count": 3, - }) + example = "https://pixhost.to/gallery/ID" def items(self): page = text.extr(self.request( @@ -307,16 +233,7 @@ class PostimgImageExtractor(ImagehostImageExtractor): category = "postimg" pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" r"\.(?:cc|org)/(?!gallery/)(?:image/)?([^/?#]+)/?)") - test = ( - ("https://postimages.org/Wtn2b3hC"), - ("https://www.postimages.org/Wtn2b3hC"), - ("https://pixxxels.cc/Wtn2b3hC"), - ("https://postimg.cc/Wtn2b3hC", { - "url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", - "keyword": "2d05808d04e4e83e33200db83521af06e3147a84", - "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", - }), - ) + example = "https://postimages.org/ID" def get_info(self, page): pos = page.index(' id="download"') @@ -331,10 +248,7 @@ class PostimgGalleryExtractor(ImagehostImageExtractor): subcategory = "gallery" pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" r"/(?:gallery/)([^/?#]+)/?)") - test = ("https://postimg.cc/gallery/wxpDLgX", { - "pattern": PostimgImageExtractor.pattern, - "count": 22, - }) + example = "https://postimages.org/gallery/ID" def items(self): page = self.request(self.page_url).text @@ -348,11 +262,7 @@ class TurboimagehostImageExtractor(ImagehostImageExtractor): category = "turboimagehost" pattern = (r"(?:https?://)?((?:www\.)?turboimagehost\.com" r"/p/(\d+)/[^/?#]+\.html)") - test = ("https://www.turboimagehost.com/p/39078423/test--.png.html", { - "url": "b94de43612318771ced924cb5085976f13b3b90e", - "keyword": "704757ca8825f51cec516ec44c1e627c1f2058ca", - "content": "f38b54b17cd7462e687b58d83f00fca88b1b105a", - }) + example = "https://www.turboimagehost.com/p/12345/NAME.EXT.html" def get_info(self, page): url = text.extract(page, 'src="', '"', page.index("<img "))[0] @@ -363,10 +273,7 @@ class ViprImageExtractor(ImagehostImageExtractor): """Extractor for single images from vipr.im""" category = "vipr" pattern = r"(?:https?://)?(vipr\.im/(\w+))" - test = ("https://vipr.im/kcd5jcuhgs3v.html", { - "url": "88f6a3ecbf3356a11ae0868b518c60800e070202", - "keyword": "c432e8a1836b0d97045195b745731c2b1bb0e771", - }) + example = "https://vipr.im/abc123.html" def get_info(self, page): url = text.extr(page, '<img src="', '"') @@ -377,11 +284,7 @@ class ImgclickImageExtractor(ImagehostImageExtractor): """Extractor for single images from imgclick.net""" category = "imgclick" pattern = r"(?:https?://)?((?:www\.)?imgclick\.net/([^/?#]+))" - test = ("http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", { - "url": "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", - "keyword": "6895256143eab955622fc149aa367777a8815ba3", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "http://imgclick.net/abc123/NAME.EXT.html" _https = False _params = "complex" @@ -395,14 +298,7 @@ class FappicImageExtractor(ImagehostImageExtractor): """Extractor for single images from fappic.com""" category = "fappic" pattern = r"(?:https?://)?((?:www\.)?fappic\.com/(\w+)/[^/?#]+)" - test = ( - ("https://fappic.com/98wxqcklyh8k/test.png"), - ("https://www.fappic.com/98wxqcklyh8k/test.png", { - "pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", - "keyword": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ) + example = "https://fappic.com/abc123/NAME.EXT" def get_info(self, page): url , pos = text.extract(page, '<a href="#"><img src="', '"') diff --git a/gallery_dl/extractor/imgbb.py b/gallery_dl/extractor/imgbb.py index ee979a65..1b741802 100644 --- a/gallery_dl/extractor/imgbb.py +++ b/gallery_dl/extractor/imgbb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -114,27 +114,7 @@ class ImgbbAlbumExtractor(ImgbbExtractor): subcategory = "album" directory_fmt = ("{category}", "{user}", "{album_name} {album_id}") pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?" - test = ( - ("https://ibb.co/album/i5PggF", { - "range": "1-80", - "url": "70afec9fcc3a6de62a6b644b487d892d8d47cf1a", - "keyword": "569e1d88ebdd27655387559cdf1cd526a3e1ab69", - }), - ("https://ibb.co/album/i5PggF?sort=title_asc", { - "range": "1-80", - "url": "afdf5fc95d8e09d77e8f44312f3e9b843987bb5a", - "keyword": "f090e14d0e5f7868595082b2c95da1309c84872d", - }), - # no user data (#471) - ("https://ibb.co/album/kYKpwF", { - "url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", - "keyword": {"user": ""}, - }), - # private - ("https://ibb.co/album/hqgWrF", { - "exception": exception.HttpError, - }), - ) + example = "https://ibb.co/album/ID" def __init__(self, match): ImgbbExtractor.__init__(self, match) @@ -169,10 +149,7 @@ class ImgbbUserExtractor(ImgbbExtractor): """Extractor for user profiles in imgbb.com""" subcategory = "user" pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?$" - test = ("https://folkie.imgbb.com", { - "range": "1-80", - "pattern": r"https?://i\.ibb\.co/\w+/[^/?#]+", - }) + example = "https://USER.imgbb.com" def __init__(self, match): ImgbbExtractor.__init__(self, match) @@ -196,19 +173,7 @@ class ImgbbUserExtractor(ImgbbExtractor): class ImgbbImageExtractor(ImgbbExtractor): subcategory = "image" pattern = r"(?:https?://)?ibb\.co/(?!album/)([^/?#]+)" - test = ("https://ibb.co/fUqh5b", { - "pattern": r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", - "content": "c5a0965178a8b357acd8aa39660092918c63795e", - "keyword": { - "id" : "fUqh5b", - "title" : "Arundel Ireeman 5", - "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", - "width" : 960, - "height": 719, - "user" : "folkie", - "extension": "jpg", - }, - }) + example = "https://ibb.co/ID" def __init__(self, match): ImgbbExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imgbox.py b/gallery_dl/extractor/imgbox.py index 530c4e1c..70697179 100644 --- a/gallery_dl/extractor/imgbox.py +++ b/gallery_dl/extractor/imgbox.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2019 Mike Fährmann +# Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from galleries at https://imgbox.com/""" +"""Extractors for https://imgbox.com/""" from .common import Extractor, Message, AsynchronousMixin from .. import text, exception @@ -63,20 +63,7 @@ class ImgboxGalleryExtractor(AsynchronousMixin, ImgboxExtractor): filename_fmt = "{num:>03}-{filename}.{extension}" archive_fmt = "{gallery_key}_{image_key}" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/g/([A-Za-z0-9]{10})" - test = ( - ("https://imgbox.com/g/JaX5V5HX7g", { - "url": "da4f15b161461119ee78841d4b8e8d054d95f906", - "keyword": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", - "content": "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", - }), - ("https://imgbox.com/g/cUGEkRbdZZ", { - "url": "76506a3aab175c456910851f66227e90484ca9f7", - "keyword": "fb0427b87983197849fb2887905e758f3e50cb6e", - }), - ("https://imgbox.com/g/JaX5V5HX7h", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgbox.com/g/12345abcde" def __init__(self, match): ImgboxExtractor.__init__(self, match) @@ -106,16 +93,7 @@ class ImgboxImageExtractor(ImgboxExtractor): subcategory = "image" archive_fmt = "{image_key}" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/([A-Za-z0-9]{8})" - test = ( - ("https://imgbox.com/qHhw7lpG", { - "url": "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", - "keyword": "dfc72310026b45f3feb4f9cada20c79b2575e1af", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("https://imgbox.com/qHhw7lpH", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgbox.com/1234abcd" def __init__(self, match): ImgboxExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imgth.py b/gallery_dl/extractor/imgth.py index 9ae22a91..3aa79229 100644 --- a/gallery_dl/extractor/imgth.py +++ b/gallery_dl/extractor/imgth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -17,24 +17,7 @@ class ImgthGalleryExtractor(GalleryExtractor): category = "imgth" root = "https://imgth.com" pattern = r"(?:https?://)?(?:www\.)?imgth\.com/gallery/(\d+)" - test = ( - ("https://imgth.com/gallery/37/wallpaper-anime", { - "url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", - "pattern": r"https://imgth\.com/images/2009/11/25" - r"/wallpaper-anime_\w+\.jpg", - "keyword": { - "count": 12, - "date": "dt:2009-11-25 18:21:00", - "extension": "jpg", - "filename": r"re:wallpaper-anime_\w+", - "gallery_id": 37, - "num": int, - "title": "Wallpaper anime", - "user": "celebrities", - }, - }), - ("https://www.imgth.com/gallery/37/wallpaper-anime"), - ) + example = "https://imgth.com/gallery/123/TITLE" def __init__(self, match): self.gallery_id = gid = match.group(1) diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index fd29a0e2..0955b4b1 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -11,7 +11,6 @@ from .common import Extractor, Message from .. import text, exception - BASE_PATTERN = r"(?:https?://)?(?:www\.|[im]\.)?imgur\.(?:com|io)" @@ -65,82 +64,7 @@ class ImgurImageExtractor(ImgurExtractor): archive_fmt = "{id}" pattern = (BASE_PATTERN + r"/(?!gallery|search)" r"(?:r/\w+/)?(\w{7}|\w{5})[sbtmlh]?") - test = ( - ("https://imgur.com/21yMxCS", { - "url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "21yMxCS", - "date" : "dt:2016-11-10 14:24:35", - "description" : "", - "downvote_count": int, - "duration" : 0, - "ext" : "png", - "favorite" : False, - "favorite_count": 0, - "has_sound" : False, - "height" : 32, - "id" : "21yMxCS", - "image_count" : 1, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : False, - "is_animated" : False, - "is_looping" : False, - "is_mature" : False, - "is_pending" : False, - "mime_type" : "image/png", - "name" : "test-テスト", - "point_count" : int, - "privacy" : "", - "score" : int, - "size" : 182, - "title" : "Test", - "upvote_count" : int, - "url" : "https://i.imgur.com/21yMxCS.png", - "view_count" : int, - "width" : 64, - }, - }), - # gifv/mp4 video - ("http://imgur.com/0gybAXR", { - "url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", - "content": "a3c080e43f58f55243ab830569ba02309d59abfc", - }), - # missing title in API response (#467) - ("https://imgur.com/XFfsmuC", { - "keyword": {"title": "Tears are a natural response to irritants"}, - }), - # animated png - ("https://imgur.com/1Nily2P", { - "pattern": "https://i.imgur.com/1Nily2P.png", - }), - # not found - ("https://imgur.com/zzzzzzz", { - "exception": exception.HttpError, - }), - ("https://m.imgur.com/r/Celebs/iHJ7tsM"), - # www - ("https://www.imgur.com/21yMxCS"), - # mobile - ("https://m.imgur.com/21yMxCS"), - # 5 character key - ("https://imgur.com/zxaY6"), - # .io - ("https://imgur.io/zxaY6"), - # direct link - ("https://i.imgur.com/21yMxCS.png"), - # direct link .io - ("https://i.imgur.io/21yMxCS.png"), - # direct link thumbnail - ("https://i.imgur.com/21yMxCSh.png"), - # direct link (short) - ("https://i.imgur.com/zxaY6.gif"), - # direct link (short; thumb) - ("https://i.imgur.com/zxaY6s.gif"), - ) + example = "https://imgur.com/abcdefg" def items(self): image = self.api.image(self.key) @@ -165,77 +89,7 @@ class ImgurAlbumExtractor(ImgurExtractor): filename_fmt = "{category}_{album[id]}_{num:>03}_{id}.{extension}" archive_fmt = "{album[id]}_{id}" pattern = BASE_PATTERN + r"/a/(\w{7}|\w{5})" - test = ( - ("https://imgur.com/a/TcBmP", { - "url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", - "keyword": { - "album": { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "693j2Kr", - "date" : "dt:2015-10-09 10:37:50", - "description" : "", - "downvote_count": 0, - "favorite" : False, - "favorite_count": 0, - "id" : "TcBmP", - "image_count" : 19, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : True, - "is_mature" : False, - "is_pending" : False, - "privacy" : "private", - "score" : int, - "title" : "138", - "upvote_count" : int, - "url" : "https://imgur.com/a/TcBmP", - "view_count" : int, - "virality" : int, - }, - "account_id" : 0, - "count" : 19, - "date" : "type:datetime", - "description": "", - "ext" : "jpg", - "has_sound" : False, - "height" : int, - "id" : str, - "is_animated": False, - "is_looping" : False, - "mime_type" : "image/jpeg", - "name" : str, - "num" : int, - "size" : int, - "title" : str, - "type" : "image", - "updated_at" : None, - "url" : str, - "width" : int, - }, - }), - # large album - ("https://imgur.com/a/eD9CT", { - "url": "de748c181a04d18bef1de9d4f4866ef0a06d632b", - }), - # 7 character album hash - ("https://imgur.com/a/RhJXhVT/all", { - "url": "695ef0c950023362a0163ee5041796300db76674", - }), - ("https://imgur.com/a/TcBmQ", { - "exception": exception.HttpError, - }), - # empty, no 'media' (#2557) - ("https://imgur.com/a/pjOnJA0", { - "count": 0, - }), - # www - ("https://www.imgur.com/a/TcBmP"), - # .io - ("https://imgur.io/a/TcBmP"), - # mobile - ("https://m.imgur.com/a/TcBmP"), - ) + example = "https://imgur.com/a/abcde" def items(self): album = self.api.album(self.key) @@ -268,19 +122,7 @@ class ImgurGalleryExtractor(ImgurExtractor): """Extractor for imgur galleries""" subcategory = "gallery" pattern = BASE_PATTERN + r"/(?:gallery|t/\w+)/(\w{7}|\w{5})" - test = ( - # non-album gallery (#380) - ("https://imgur.com/gallery/zf2fIms", { - "pattern": "https://imgur.com/zf2fIms", - }), - ("https://imgur.com/gallery/eD9CT", { - "pattern": "https://imgur.com/a/eD9CT", - }), - ("https://imgur.com/t/unmuted/26sEhNr"), - ("https://imgur.com/t/cat/qSB8NbN"), - # .io - ("https://imgur.io/t/cat/qSB8NbN"), - ) + example = "https://imgur.com/gallery/abcde" def items(self): if self.api.gallery(self.key)["is_album"]: @@ -296,15 +138,7 @@ class ImgurUserExtractor(ImgurExtractor): """Extractor for all images posted by a user""" subcategory = "user" pattern = BASE_PATTERN + r"/user/([^/?#]+)(?:/posts|/submitted)?/?$" - test = ( - ("https://imgur.com/user/Miguenzo", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }), - ("https://imgur.com/user/Miguenzo/posts"), - ("https://imgur.com/user/Miguenzo/submitted"), - ) + example = "https://imgur.com/user/USER" def items(self): return self._items_queue(self.api.account_submissions(self.key)) @@ -314,11 +148,7 @@ class ImgurFavoriteExtractor(ImgurExtractor): """Extractor for a user's favorites""" subcategory = "favorite" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/?$" - test = ("https://imgur.com/user/Miguenzo/favorites", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/user/USER/favorites" def items(self): return self._items_queue(self.api.account_favorites(self.key)) @@ -328,16 +158,7 @@ class ImgurFavoriteFolderExtractor(ImgurExtractor): """Extractor for a user's favorites folder""" subcategory = "favorite-folder" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/folder/(\d+)" - test = ( - ("https://imgur.com/user/mikf1/favorites/folder/11896757/public", { - "pattern": r"https://imgur\.com(/a)?/\w+$", - "count": 3, - }), - ("https://imgur.com/user/mikf1/favorites/folder/11896741/private", { - "pattern": r"https://imgur\.com(/a)?/\w+$", - "count": 5, - }), - ) + example = "https://imgur.com/user/USER/favorites/folder/12345/TITLE" def __init__(self, match): ImgurExtractor.__init__(self, match) @@ -352,11 +173,7 @@ class ImgurSubredditExtractor(ImgurExtractor): """Extractor for a subreddits's imgur links""" subcategory = "subreddit" pattern = BASE_PATTERN + r"/r/([^/?#]+)/?$" - test = ("https://imgur.com/r/pics", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/r/SUBREDDIT" def items(self): return self._items_queue(self.api.gallery_subreddit(self.key)) @@ -366,11 +183,7 @@ class ImgurTagExtractor(ImgurExtractor): """Extractor for imgur tag searches""" subcategory = "tag" pattern = BASE_PATTERN + r"/t/([^/?#]+)$" - test = ("https://imgur.com/t/animals", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/t/TAG" def items(self): return self._items_queue(self.api.gallery_tag(self.key)) @@ -380,11 +193,7 @@ class ImgurSearchExtractor(ImgurExtractor): """Extractor for imgur search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/[^?#]+)?/?\?q=([^&#]+)" - test = ("https://imgur.com/search?q=cute+cat", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/search?q=UERY" def items(self): key = text.unquote(self.key.replace("+", " ")) diff --git a/gallery_dl/extractor/inkbunny.py b/gallery_dl/extractor/inkbunny.py index c6df16b0..1a4cf21f 100644 --- a/gallery_dl/extractor/inkbunny.py +++ b/gallery_dl/extractor/inkbunny.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -72,51 +72,7 @@ class InkbunnyUserExtractor(InkbunnyExtractor): """Extractor for inkbunny user profiles""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!s/)(gallery/|scraps/)?(\w+)(?:$|[/?#])" - test = ( - ("https://inkbunny.net/soina", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/\d+/\d+_soina_.+", - "range": "20-50", - "keyword": { - "date" : "type:datetime", - "deleted" : bool, - "file_id" : "re:[0-9]+", - "filename" : r"re:[0-9]+_soina_\w+", - "full_file_md5": "re:[0-9a-f]{32}", - "mimetype" : str, - "submission_id": "re:[0-9]+", - "user_id" : "20969", - "comments_count" : "re:[0-9]+", - "deleted" : bool, - "favorite" : bool, - "favorites_count": "re:[0-9]+", - "friends_only" : bool, - "guest_block" : bool, - "hidden" : bool, - "pagecount" : "re:[0-9]+", - "pools" : list, - "pools_count" : int, - "public" : bool, - "rating_id" : "re:[0-9]+", - "rating_name" : str, - "ratings" : list, - "scraps" : bool, - "tags" : list, - "title" : str, - "type_name" : str, - "username" : "soina", - "views" : str, - }, - }), - ("https://inkbunny.net/gallery/soina", { - "range": "1-25", - "keyword": {"scraps": False}, - }), - ("https://inkbunny.net/scraps/soina", { - "range": "1-25", - "keyword": {"scraps": True}, - }), - ) + example = "https://inkbunny.net/USER" def __init__(self, match): kind, self.user = match.groups() @@ -148,14 +104,7 @@ class InkbunnyPoolExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"poolview_process\.php\?pool_id=(\d+)|" r"submissionsviewall\.php\?([^#]+&mode=pool&[^#]+))") - test = ( - ("https://inkbunny.net/poolview_process.php?pool_id=28985", { - "count": 9, - "keyword": {"pool_id": "28985"}, - }), - ("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no"), - ) + example = "https://inkbunny.net/poolview_process.php?pool_id=12345" def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -185,16 +134,8 @@ class InkbunnyFavoriteExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"userfavorites_process\.php\?favs_user_id=(\d+)|" r"submissionsviewall\.php\?([^#]+&mode=userfavs&[^#]+))") - test = ( - ("https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/\d+/\d+_\w+_.+", - "range": "20-50", - "keyword": {"favs_user_id": "20969"}, - }), - ("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969"), - ) + example = ("https://inkbunny.net/userfavorites_process.php" + "?mode=userfavs&favs_user_id=12345") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -225,26 +166,8 @@ class InkbunnySearchExtractor(InkbunnyExtractor): subcategory = "search" pattern = (BASE_PATTERN + r"/submissionsviewall\.php\?([^#]+&mode=search&[^#]+)") - test = (("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=search&page=1&orderby=create_datetime&text=cute" - "&stringtype=and&keywords=yes&title=yes&description=no&artist=" - "&favsby=&type=&days=&keyword_id=&user_id=&random=&md5="), { - "range": "1-10", - "count": 10, - "keyword": { - "search": { - "rid": "ffffffffff", - "mode": "search", - "page": "1", - "orderby": "create_datetime", - "text": "cute", - "stringtype": "and", - "keywords": "yes", - "title": "yes", - "description": "no", - }, - }, - }) + example = ("https://inkbunny.net/submissionsviewall.php" + "?mode=search&text=TAG") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -279,15 +202,8 @@ class InkbunnyFollowingExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"watchlist_process\.php\?mode=watching&user_id=(\d+)|" r"usersviewall\.php\?([^#]+&mode=watching&[^#]+))") - test = ( - (("https://inkbunny.net/watchlist_process.php" - "?mode=watching&user_id=20969"), { - "pattern": InkbunnyUserExtractor.pattern, - "count": ">= 90", - }), - ("https://inkbunny.net/usersviewall.php?rid=ffffffffff" - "&mode=watching&page=1&user_id=20969&orderby=added&namesonly="), - ) + example = ("https://inkbunny.net/watchlist_process.php" + "?mode=watching&user_id=12345") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -324,16 +240,7 @@ class InkbunnyPostExtractor(InkbunnyExtractor): """Extractor for individual Inkbunny posts""" subcategory = "post" pattern = BASE_PATTERN + r"/s/(\d+)" - test = ( - ("https://inkbunny.net/s/1829715", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/2626/2626843_soina_dscn2296\.jpg", - "content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", - }), - ("https://inkbunny.net/s/2044094", { - "count": 4, - }), - ) + example = "https://inkbunny.net/s/12345" def __init__(self, match): InkbunnyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index d9049678..10a1c7bf 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -398,11 +398,7 @@ class InstagramUserExtractor(InstagramExtractor): """Extractor for an Instagram user profile""" subcategory = "user" pattern = USER_PATTERN + r"/?(?:$|[?#])" - test = ( - ("https://www.instagram.com/instagram/"), - ("https://www.instagram.com/instagram/?hl=en"), - ("https://www.instagram.com/id:25025320/"), - ) + example = "https://www.instagram.com/USER/" def initialize(self): pass @@ -427,10 +423,7 @@ class InstagramPostsExtractor(InstagramExtractor): """Extractor for an Instagram user's posts""" subcategory = "posts" pattern = USER_PATTERN + r"/posts" - test = ("https://www.instagram.com/instagram/posts/", { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/USER/posts/" def posts(self): uid = self.api.user_id(self.item) @@ -441,10 +434,7 @@ class InstagramReelsExtractor(InstagramExtractor): """Extractor for an Instagram user's reels""" subcategory = "reels" pattern = USER_PATTERN + r"/reels" - test = ("https://www.instagram.com/instagram/reels/", { - "range": "40-60", - "count": ">= 20", - }) + example = "https://www.instagram.com/USER/reels/" def posts(self): uid = self.api.user_id(self.item) @@ -455,15 +445,7 @@ class InstagramTaggedExtractor(InstagramExtractor): """Extractor for an Instagram user's tagged posts""" subcategory = "tagged" pattern = USER_PATTERN + r"/tagged" - test = ("https://www.instagram.com/instagram/tagged/", { - "range": "1-16", - "count": ">= 16", - "keyword": { - "tagged_owner_id" : "25025320", - "tagged_username" : "instagram", - "tagged_full_name": "Instagram", - }, - }) + example = "https://www.instagram.com/USER/tagged/" def metadata(self): if self.item.startswith("id:"): @@ -487,11 +469,7 @@ class InstagramGuideExtractor(InstagramExtractor): """Extractor for an Instagram guide""" subcategory = "guide" pattern = USER_PATTERN + r"/guide/[^/?#]+/(\d+)" - test = (("https://www.instagram.com/kadakaofficial/guide" - "/knit-i-need-collection/18131821684305217/"), { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/USER/guide/" def __init__(self, match): InstagramExtractor.__init__(self, match) @@ -508,10 +486,7 @@ class InstagramSavedExtractor(InstagramExtractor): """Extractor for an Instagram user's saved media""" subcategory = "saved" pattern = USER_PATTERN + r"/saved(?:/all-posts)?/?$" - test = ( - ("https://www.instagram.com/instagram/saved/"), - ("https://www.instagram.com/instagram/saved/all-posts/"), - ) + example = "https://www.instagram.com/USER/saved/" def posts(self): return self.api.user_saved() @@ -521,9 +496,7 @@ class InstagramCollectionExtractor(InstagramExtractor): """Extractor for Instagram collection""" subcategory = "collection" pattern = USER_PATTERN + r"/saved/([^/?#]+)/([^/?#]+)" - test = ( - "https://www.instagram.com/instagram/saved/collection_name/123456789/", - ) + example = "https://www.instagram.com/USER/saved/COLLECTION/12345" def __init__(self, match): InstagramExtractor.__init__(self, match) @@ -545,14 +518,7 @@ class InstagramStoriesExtractor(InstagramExtractor): pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" r"/s(?:tories/(?:highlights/(\d+)|([^/?#]+)(?:/(\d+))?)" r"|/(aGlnaGxpZ2h0[^?#]+)(?:\?story_media_id=(\d+))?)") - test = ( - ("https://www.instagram.com/stories/instagram/"), - ("https://www.instagram.com/stories/highlights/18042509488170095/"), - ("https://instagram.com/stories/geekmig/2724343156064789461"), - ("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1"), - ("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1" - "?story_media_id=2724343156064789461"), - ) + example = "https://www.instagram.com/stories/USER/" def __init__(self, match): h1, self.user, m1, h2, m2 = match.groups() @@ -587,7 +553,7 @@ class InstagramHighlightsExtractor(InstagramExtractor): """Extractor for an Instagram user's story highlights""" subcategory = "highlights" pattern = USER_PATTERN + r"/highlights" - test = ("https://www.instagram.com/instagram/highlights",) + example = "https://www.instagram.com/USER/highlights/" def posts(self): uid = self.api.user_id(self.item) @@ -598,10 +564,7 @@ class InstagramFollowingExtractor(InstagramExtractor): """Extractor for an Instagram user's followed users""" subcategory = "following" pattern = USER_PATTERN + r"/following" - test = ("https://www.instagram.com/instagram/following", { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/USER/following/" def items(self): uid = self.api.user_id(self.item) @@ -616,10 +579,7 @@ class InstagramTagExtractor(InstagramExtractor): subcategory = "tag" directory_fmt = ("{category}", "{subcategory}", "{tag}") pattern = BASE_PATTERN + r"/explore/tags/([^/?#]+)" - test = ("https://www.instagram.com/explore/tags/instagram/", { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/explore/tags/TAG/" def metadata(self): return {"tag": text.unquote(self.item)} @@ -632,10 +592,7 @@ class InstagramAvatarExtractor(InstagramExtractor): """Extractor for an Instagram user's avatar""" subcategory = "avatar" pattern = USER_PATTERN + r"/avatar" - test = ("https://www.instagram.com/instagram/avatar", { - "pattern": r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19" - r"/281440578_1088265838702675_6233856337905829714_n\.jpg", - }) + example = "https://www.instagram.com/USER/avatar/" def posts(self): if self._logged_in: @@ -675,102 +632,7 @@ class InstagramPostExtractor(InstagramExtractor): subcategory = "post" pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" r"/(?:[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)") - test = ( - # GraphImage - ("https://www.instagram.com/p/BqvsDleB3lV/", { - "pattern": r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)" - r"/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35" - r"/44877605_725955034447492_3123079845831750529_n.jpg", - "keyword": { - "date": "dt:2018-11-29 01:04:04", - "description": str, - "height": int, - "likes": int, - "location_id": "214424288", - "location_slug": "hong-kong", - "location_url": "re:/explore/locations/214424288/hong-kong/", - "media_id": "1922949326347663701", - "shortcode": "BqvsDleB3lV", - "post_id": "1922949326347663701", - "post_shortcode": "BqvsDleB3lV", - "post_url": "https://www.instagram.com/p/BqvsDleB3lV/", - "tags": ["#WHPsquares"], - "typename": "GraphImage", - "username": "instagram", - "width": int, - } - }), - # GraphSidecar - ("https://www.instagram.com/p/BoHk1haB5tM/", { - "count": 5, - "keyword": { - "sidecar_media_id": "1875629777499953996", - "sidecar_shortcode": "BoHk1haB5tM", - "post_id": "1875629777499953996", - "post_shortcode": "BoHk1haB5tM", - "post_url": "https://www.instagram.com/p/BoHk1haB5tM/", - "num": int, - "likes": int, - "username": "instagram", - } - }), - # GraphVideo - ("https://www.instagram.com/p/Bqxp0VSBgJg/", { - "pattern": r"/46840863_726311431074534_7805566102611403091_n\.mp4", - "keyword": { - "date": "dt:2018-11-29 19:23:58", - "description": str, - "height": int, - "likes": int, - "media_id": "1923502432034620000", - "post_url": "https://www.instagram.com/p/Bqxp0VSBgJg/", - "shortcode": "Bqxp0VSBgJg", - "tags": ["#ASMR"], - "typename": "GraphVideo", - "username": "instagram", - "width": int, - } - }), - # GraphVideo (IGTV) - ("https://www.instagram.com/tv/BkQjCfsBIzi/", { - "pattern": r"/10000000_597132547321814_702169244961988209_n\.mp4", - "keyword": { - "date": "dt:2018-06-20 19:51:32", - "description": str, - "height": int, - "likes": int, - "media_id": "1806097553666903266", - "post_url": "https://www.instagram.com/p/BkQjCfsBIzi/", - "shortcode": "BkQjCfsBIzi", - "typename": "GraphVideo", - "username": "instagram", - "width": int, - } - }), - # GraphSidecar with 2 embedded GraphVideo objects - ("https://www.instagram.com/p/BtOvDOfhvRr/", { - "count": 2, - "keyword": { - "post_url": "https://www.instagram.com/p/BtOvDOfhvRr/", - "sidecar_media_id": "1967717017113261163", - "sidecar_shortcode": "BtOvDOfhvRr", - "video_url": str, - } - }), - # GraphImage with tagged user - ("https://www.instagram.com/p/B_2lf3qAd3y/", { - "keyword": { - "tagged_users": [{ - "id" : "1246468638", - "username" : "kaaymbl", - "full_name": "Call Me Kay", - }] - } - }), - # URL with username (#2085) - ("https://www.instagram.com/dm/p/CW042g7B9CY/"), - ("https://www.instagram.com/reel/CDg_6Y1pxWu/"), - ) + example = "https://www.instagram.com/p/abcdefg/" def posts(self): return self.api.media(self.item) diff --git a/gallery_dl/extractor/issuu.py b/gallery_dl/extractor/issuu.py index 558c4fd7..f6170c28 100644 --- a/gallery_dl/extractor/issuu.py +++ b/gallery_dl/extractor/issuu.py @@ -26,31 +26,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor): filename_fmt = "{num:>03}.{extension}" archive_fmt = "{document[publicationId]}_{num}" pattern = r"(?:https?://)?issuu\.com(/[^/?#]+/docs/[^/?#]+)" - test = ("https://issuu.com/issuu/docs/motions-1-2019/", { - "pattern": r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", - "count" : 36, - "keyword": { - "document": { - "access" : "PUBLIC", - "contentRating" : { - "isAdsafe" : True, - "isExplicit": False, - "isReviewed": True, - }, - "date" : "dt:2019-09-16 00:00:00", - "description" : "re:Motions, the brand new publication by I", - "documentName" : "motions-1-2019", - "downloadable" : False, - "pageCount" : 36, - "publicationId" : "d99ec95935f15091b040cb8060f05510", - "title" : "Motions by Issuu - Issue 1", - "username" : "issuu", - }, - "extension": "jpg", - "filename" : r"re:page_\d+", - "num" : int, - }, - }) + example = "https://issuu.com/issuu/docs/TITLE/" def metadata(self, page): data = util.json_loads(text.rextract( @@ -78,10 +54,7 @@ class IssuuUserExtractor(IssuuBase, Extractor): """Extractor for all publications of a user/publisher""" subcategory = "user" pattern = r"(?:https?://)?issuu\.com/([^/?#]+)/?$" - test = ("https://issuu.com/issuu", { - "pattern": IssuuPublicationExtractor.pattern, - "count" : "> 25", - }) + example = "https://issuu.com/USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/itaku.py b/gallery_dl/extractor/itaku.py index 356a002b..e9ec978b 100644 --- a/gallery_dl/extractor/itaku.py +++ b/gallery_dl/extractor/itaku.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -63,12 +63,7 @@ class ItakuGalleryExtractor(ItakuExtractor): """Extractor for posts from an itaku user gallery""" subcategory = "gallery" pattern = BASE_PATTERN + r"/profile/([^/?#]+)/gallery" - test = ("https://itaku.ee/profile/piku/gallery", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs" - r"/[^/?#]+\.(jpg|png|gif)", - "range": "1-10", - "count": 10, - }) + example = "https://itaku.ee/profile/USER/gallery" def posts(self): return self.api.galleries_images(self.item) @@ -77,62 +72,7 @@ class ItakuGalleryExtractor(ItakuExtractor): class ItakuImageExtractor(ItakuExtractor): subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" - test = ( - ("https://itaku.ee/images/100471", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs" - r"/220504_oUNIAFT\.png", - "count": 1, - "keyword": { - "already_pinned": None, - "blacklisted": { - "blacklisted_tags": [], - "is_blacklisted": False - }, - "can_reshare": True, - "date": "dt:2022-05-05 19:21:17", - "date_added": "2022-05-05T19:21:17.674148Z", - "date_edited": "2022-05-25T14:37:46.220612Z", - "description": "sketch from drawpile", - "extension": "png", - "filename": "220504_oUNIAFT", - "hotness_score": float, - "id": 100471, - "image": "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs" - "/220504_oUNIAFT.png", - "image_xl": "https://d1wmr8tlk3viaj.cloudfront.net" - "/gallery_imgs/220504_oUNIAFT/lg.jpg", - "liked_by_you": False, - "maturity_rating": "SFW", - "num_comments": int, - "num_likes": int, - "num_reshares": int, - "obj_tags": 136446, - "owner": 16775, - "owner_avatar": "https://d1wmr8tlk3viaj.cloudfront.net" - "/profile_pics/av2022r_vKYVywc/md.jpg", - "owner_displayname": "Piku", - "owner_username": "piku", - "reshared_by_you": False, - "sections": ["Fanart/Miku"], - "tags": list, - "tags_character": ["hatsune_miku"], - "tags_copyright": ["vocaloid"], - "tags_general" : ["twintails", "green_hair", "flag", - "gloves", "green_eyes", "female", - "racing_miku"], - "title": "Racing Miku 2022 Ver.", - "too_mature": False, - "uncompressed_filesize": "0.62", - "video": None, - "visibility": "PUBLIC", - }, - }), - # video - ("https://itaku.ee/images/19465", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids" - r"/sleepy_af_OY5GHWw\.mp4", - }), - ) + example = "https://itaku.ee/images/12345" def posts(self): return (self.api.image(self.item),) diff --git a/gallery_dl/extractor/itchio.py b/gallery_dl/extractor/itchio.py index 96ebbdc4..799dd66e 100644 --- a/gallery_dl/extractor/itchio.py +++ b/gallery_dl/extractor/itchio.py @@ -21,28 +21,7 @@ class ItchioGameExtractor(Extractor): filename_fmt = "{game[title]} ({id}).{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?(\w+).itch\.io/([\w-]+)" - test = ( - ("https://sirtartarus.itch.io/a-craft-of-mine", { - "pattern": r"https://\w+\.ssl\.hwcdn\.net/upload2" - r"/game/1983311/7723751\?", - "count": 1, - "keyword": { - "extension": "", - "filename": "7723751", - "game": { - "id": 1983311, - "noun": "game", - "title": "A Craft Of Mine", - "url": "https://sirtartarus.itch.io/a-craft-of-mine", - }, - "user": { - "id": 4060052, - "name": "SirTartarus", - "url": "https://sirtartarus.itch.io", - }, - }, - }), - ) + example = "https://USER.itch.io/GAME" def __init__(self, match): self.user, self.slug = match.groups() diff --git a/gallery_dl/extractor/jpgfish.py b/gallery_dl/extractor/jpgfish.py index a63acb55..8862a7b7 100644 --- a/gallery_dl/extractor/jpgfish.py +++ b/gallery_dl/extractor/jpgfish.py @@ -35,29 +35,7 @@ class JpgfishImageExtractor(JpgfishExtractor): """Extractor for jpgfish Images""" subcategory = "image" pattern = BASE_PATTERN + r"/img/((?:[^/?#]+\.)?(\w+))" - test = ( - ("https://jpg1.su/img/funnymeme.LecXGS", { - "pattern": r"https://simp3\.jpg\.church/images/funnymeme\.jpg", - "content": "098e5e9b17ad634358426e0ffd1c93871474d13c", - "keyword": { - "album": "", - "extension": "jpg", - "filename": "funnymeme", - "id": "LecXGS", - "url": "https://simp3.jpg.church/images/funnymeme.jpg", - "user": "exearco", - }, - }), - ("https://jpg.church/img/auCruA", { - "pattern": r"https://simp2\.jpg\.church/hannahowo_00457\.jpg", - "keyword": {"album": "401-500"}, - }), - ("https://jpeg.pet/img/funnymeme.LecXGS"), - ("https://jpg.pet/img/funnymeme.LecXGS"), - ("https://jpg.fishing/img/funnymeme.LecXGS"), - ("https://jpg.fish/img/funnymeme.LecXGS"), - ("https://jpg.church/img/funnymeme.LecXGS"), - ) + example = "https://jpg1.su/img/TITLE.ID" def __init__(self, match): JpgfishExtractor.__init__(self, match) @@ -84,22 +62,7 @@ class JpgfishAlbumExtractor(JpgfishExtractor): """Extractor for jpgfish Albums""" subcategory = "album" pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?" - test = ( - ("https://jpg1.su/album/CDilP/?sort=date_desc&page=1", { - "count": 2, - }), - ("https://jpg.fishing/a/gunggingnsk.N9OOI", { - "count": 114, - }), - ("https://jpg.fish/a/101-200.aNJ6A/", { - "count": 100, - }), - ("https://jpg.church/a/hannahowo.aNTdH/sub", { - "count": 606, - }), - ("https://jpeg.pet/album/CDilP/?sort=date_desc&page=1"), - ("https://jpg.pet/album/CDilP/?sort=date_desc&page=1"), - ) + example = "https://jpg1.su/album/TITLE.ID" def __init__(self, match): JpgfishExtractor.__init__(self, match) @@ -123,19 +86,7 @@ class JpgfishUserExtractor(JpgfishExtractor): """Extractor for jpgfish Users""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?" - test = ( - ("https://jpg1.su/exearco", { - "count": 3, - }), - ("https://jpg.church/exearco/albums", { - "count": 1, - }), - ("https://jpeg.pet/exearco"), - ("https://jpg.pet/exearco"), - ("https://jpg.fishing/exearco"), - ("https://jpg.fish/exearco"), - ("https://jpg.church/exearco"), - ) + example = "https://jpg1.su/USER" def __init__(self, match): JpgfishExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/jschan.py b/gallery_dl/extractor/jschan.py index fe758faa..398256dc 100644 --- a/gallery_dl/extractor/jschan.py +++ b/gallery_dl/extractor/jschan.py @@ -31,12 +31,7 @@ class JschanThreadExtractor(JschanExtractor): filename_fmt = "{postId}{num:?-//} {filename}.{extension}" archive_fmt = "{board}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)\.html" - test = ( - ("https://94chan.org/art/thread/25.html", { - "pattern": r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", - "count": ">= 15" - }) - ) + example = "https://94chan.org/a/thread/12345.html" def __init__(self, match): JschanExtractor.__init__(self, match) @@ -71,15 +66,7 @@ class JschanBoardExtractor(JschanExtractor): subcategory = "board" pattern = (BASE_PATTERN + r"/([^/?#]+)" r"(?:/index\.html|/catalog\.html|/\d+\.html|/?$)") - test = ( - ("https://94chan.org/art/", { - "pattern": JschanThreadExtractor.pattern, - "count": ">= 30" - }), - ("https://94chan.org/art/2.html"), - ("https://94chan.org/art/catalog.html"), - ("https://94chan.org/art/index.html"), - ) + example = "https://94chan.org/a/" def __init__(self, match): JschanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/kabeuchi.py b/gallery_dl/extractor/kabeuchi.py index f172dcfa..867f0da8 100644 --- a/gallery_dl/extractor/kabeuchi.py +++ b/gallery_dl/extractor/kabeuchi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,16 +21,7 @@ class KabeuchiUserExtractor(Extractor): archive_fmt = "{id}_{num}" root = "https://kabe-uchiroom.com" pattern = r"(?:https?://)?kabe-uchiroom\.com/mypage/?\?id=(\d+)" - test = ( - ("https://kabe-uchiroom.com/mypage/?id=919865303848255493", { - "pattern": (r"https://kabe-uchiroom\.com/accounts/upfile/3/" - r"919865303848255493/\w+\.jpe?g"), - "count": ">= 24", - }), - ("https://kabe-uchiroom.com/mypage/?id=123456789", { - "exception": exception.NotFoundError, - }), - ) + example = "https://kabe-uchiroom.com/mypage/?id=12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/keenspot.py b/gallery_dl/extractor/keenspot.py index 78454a6f..b8ecc143 100644 --- a/gallery_dl/extractor/keenspot.py +++ b/gallery_dl/extractor/keenspot.py @@ -20,33 +20,7 @@ class KeenspotComicExtractor(Extractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{comic}_{filename}" pattern = r"(?:https?://)?(?!www\.|forums\.)([\w-]+)\.keenspot\.com(/.+)?" - test = ( - # link - ("http://marksmen.keenspot.com/", { - "range": "1-3", - "url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", - }), - # id - ("http://barkercomic.keenspot.com/", { - "range": "1-3", - "url": "c4080926db18d00bac641fdd708393b7d61379e6", - }), - # id v2 - ("http://crowscare.keenspot.com/", { - "range": "1-3", - "url": "a00e66a133dd39005777317da90cef921466fcaa" - }), - # ks - ("http://supernovas.keenspot.com/", { - "range": "1-3", - "url": "de21b12887ef31ff82edccbc09d112e3885c3aab" - }), - # "random" access - ("http://twokinds.keenspot.com/comic/1066/", { - "range": "1-3", - "url": "6a784e11370abfb343dcad9adbb7718f9b7be350", - }) - ) + example = "http://COMIC.keenspot.com/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 2ed73e9c..0034e690 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -216,19 +216,7 @@ class KemonopartyUserExtractor(KemonopartyExtractor): """Extractor for all posts from a kemono.party user listing""" subcategory = "user" pattern = USER_PATTERN + r"/?(?:\?o=(\d+))?(?:$|[?#])" - test = ( - ("https://kemono.party/fanbox/user/6993449", { - "range": "1-25", - "count": 25, - }), - # 'max-posts' option, 'o' query parameter (#1674) - ("https://kemono.party/patreon/user/881792?o=150", { - "options": (("max-posts", 25),), - "count": "< 100", - }), - ("https://kemono.su/subscribestar/user/alcorart"), - ("https://kemono.party/subscribestar/user/alcorart"), - ) + example = "https://kemono.party/SERVICE/user/12345" def __init__(self, match): _, _, service, user_id, offset = match.groups() @@ -256,87 +244,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor): """Extractor for a single kemono.party post""" subcategory = "post" pattern = USER_PATTERN + r"/post/([^/?#]+)" - test = ( - ("https://kemono.party/fanbox/user/6993449/post/506575", { - "pattern": r"https://kemono.party/data/21/0f" - r"/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", - "content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", - "keyword": { - "added": "Wed, 06 May 2020 20:28:02 GMT", - "content": str, - "count": 1, - "date": "dt:2019-08-11 02:09:04", - "edited": None, - "embed": dict, - "extension": "jpeg", - "filename": "P058kDFYus7DbqAkGlfWTlOr", - "hash": "210f35388e28bbcf756db18dd516e2d8" - "2ce758e0d32881eeee76d43e1716d382", - "id": "506575", - "num": 1, - "published": "Sun, 11 Aug 2019 02:09:04 GMT", - "service": "fanbox", - "shared_file": False, - "subcategory": "fanbox", - "title": "c96取り置き", - "type": "file", - "user": "6993449", - }, - }), - # inline image (#1286) - ("https://kemono.party/fanbox/user/7356311/post/802343", { - "pattern": r"https://kemono\.party/data/47/b5/47b5c014ecdcfabdf2c8" - r"5eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", - "keyword": {"hash": "47b5c014ecdcfabdf2c85eec53f1133a" - "76336997ae8596f332e97d956a460ad2"}, - }), - # kemono.party -> data.kemono.party - ("https://kemono.party/gumroad/user/trylsc/post/IURjT", { - "pattern": r"https://kemono\.party/data/(" - r"a4/7b/a47bfe938d8c1682eef06e885927484cd8df1b.+\.jpg|" - r"c6/04/c6048f5067fd9dbfa7a8be565ac194efdfb6e4.+\.zip)", - }), - # username (#1548, #1652) - ("https://kemono.party/gumroad/user/3252870377455/post/aJnAH", { - "options": (("metadata", True),), - "keyword": {"username": "Kudalyn's Creations"}, - }), - # skip patreon duplicates - ("https://kemono.party/patreon/user/4158582/post/32099982", { - "count": 2, - }), - # allow duplicates (#2440) - ("https://kemono.party/patreon/user/4158582/post/32099982", { - "options": (("duplicates", True),), - "count": 3, - }), - # DMs (#2008) - ("https://kemono.party/patreon/user/34134344/post/38129255", { - "options": (("dms", True),), - "keyword": {"dms": [{ - "body": r"re:Hi! Thank you very much for supporting the work I" - r" did in May. Here's your reward pack! I hope you fin" - r"d something you enjoy in it. :\)\n\nhttps://www.medi" - r"afire.com/file/\w+/Set13_tier_2.zip/file", - "date": "2021-07-31 02:47:51.327865", - }]}, - }), - # coomer.party (#2100) - ("https://coomer.party/onlyfans/user/alinity/post/125962203", { - "pattern": r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968" - r"c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg", - }), - # invalid file (#3510) - ("https://kemono.party/patreon/user/19623797/post/29035449", { - "pattern": r"907ba78b4545338d3539683e63ecb51c" - r"f51c10adc9dabd86e92bd52339f298b9\.txt", - "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", # empty - }), - ("https://kemono.su/subscribestar/user/alcorart/post/184330"), - ("https://kemono.party/subscribestar/user/alcorart/post/184330"), - ("https://www.kemono.party/subscribestar/user/alcorart/post/184330"), - ("https://beta.kemono.party/subscribestar/user/alcorart/post/184330"), - ) + example = "https://kemono.party/SERVICE/user/12345/post/12345" def __init__(self, match): _, _, service, user_id, post_id = match.groups() @@ -359,30 +267,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): filename_fmt = "{id}_{num:>02}_{filename}.{extension}" archive_fmt = "discord_{server}_{id}_{num}" pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)" - test = ( - (("https://kemono.party/discord" - "/server/488668827274444803#finish-work"), { - "count": 4, - "keyword": {"channel_name": "finish-work"}, - }), - (("https://kemono.su/discord" - "/server/256559665620451329/channel/462437519519383555#"), { - "pattern": r"https://kemono\.su/data/(" - r"e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|" - r"51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", - "keyword": {"hash": "re:e377e3525164559484ace2e64425b0cec1db08" - "|51453640a5e0a4d23fbf57fb85390f9c5ec154"}, - "count": ">= 2", - }), - # 'inline' files - (("https://kemono.party/discord" - "/server/315262215055736843/channel/315262215055736843#general"), { - "pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", - "options": (("image-filter", "type == 'inline'"),), - "keyword": {"hash": ""}, - "range": "1-5", - }), - ) + example = "https://kemono.party/discard/server/12345/channel/12345" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -461,16 +346,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): class KemonopartyDiscordServerExtractor(KemonopartyExtractor): subcategory = "discord-server" pattern = BASE_PATTERN + r"/discord/server/(\d+)$" - test = ( - ("https://kemono.party/discord/server/488668827274444803", { - "pattern": KemonopartyDiscordExtractor.pattern, - "count": 13, - }), - ("https://kemono.su/discord/server/488668827274444803", { - "pattern": KemonopartyDiscordExtractor.pattern, - "count": 13, - }), - ) + example = "https://kemono.party/discard/server/12345" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -492,23 +368,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): """Extractor for kemono.party favorites""" subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?" - test = ( - ("https://kemono.party/favorites", { - "pattern": KemonopartyUserExtractor.pattern, - "url": "f4b5b796979bcba824af84206578c79101c7f0e1", - "count": 3, - }), - ("https://kemono.party/favorites?type=post", { - "pattern": KemonopartyPostExtractor.pattern, - "url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f", - "count": 3, - }), - ("https://kemono.su/favorites?type=post", { - "pattern": KemonopartyPostExtractor.pattern, - "url": "4be8e84cb384a907a8e7997baaf6287b451783b5", - "count": 3, - }), - ) + example = "https://kemono.party/favorites" def __init__(self, match): KemonopartyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/khinsider.py b/gallery_dl/extractor/khinsider.py index 0c3b002f..d0c9c301 100644 --- a/gallery_dl/extractor/khinsider.py +++ b/gallery_dl/extractor/khinsider.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -16,30 +16,13 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor): """Extractor for soundtracks from khinsider.com""" category = "khinsider" subcategory = "soundtrack" + root = "https://downloads.khinsider.com" directory_fmt = ("{category}", "{album[name]}") archive_fmt = "{filename}.{extension}" pattern = (r"(?:https?://)?downloads\.khinsider\.com" r"/game-soundtracks/album/([^/?#]+)") - root = "https://downloads.khinsider.com" - test = (("https://downloads.khinsider.com" - "/game-soundtracks/album/horizon-riders-wii"), { - "pattern": r"https?://vgm(site|downloads)\.com" - r"/soundtracks/horizon-riders-wii/[^/]+" - r"/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", - "keyword": { - "album": { - "count": 1, - "date": "Sep 18th, 2016", - "name": "Horizon Riders", - "platform": "Wii", - "size": 26214400, - "type": "Gamerip", - }, - "extension": "mp3", - "filename": "Horizon Riders Wii - Full Soundtrack", - }, - "count": 1, - }) + example = ("https://downloads.khinsider.com" + "/game-soundtracks/album/TITLE") def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/komikcast.py b/gallery_dl/extractor/komikcast.py index 04373c4b..635b5699 100644 --- a/gallery_dl/extractor/komikcast.py +++ b/gallery_dl/extractor/komikcast.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2022 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -48,18 +48,7 @@ class KomikcastBase(): class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor): """Extractor for manga-chapters from komikcast.site""" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/)" - test = ( - (("https://komikcast.site/chapter" - "/apotheosis-chapter-02-2-bahasa-indonesia/"), { - "url": "f6b43fbc027697749b3ea1c14931c83f878d7936", - "keyword": "f3938e1aff9ad1f302f52447e9781b21f6da26d4", - }), - (("https://komikcast.me/chapter" - "/soul-land-ii-chapter-300-1-bahasa-indonesia/"), { - "url": "efd00a9bd95461272d51990d7bc54b79ff3ff2e6", - "keyword": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", - }), - ) + example = "https://komikcast.site/chapter/TITLE" def metadata(self, page): info = text.extr(page, "<title>", " - Komikcast<") @@ -79,13 +68,7 @@ class KomikcastMangaExtractor(KomikcastBase, MangaExtractor): """Extractor for manga from komikcast.site""" chapterclass = KomikcastChapterExtractor pattern = BASE_PATTERN + r"(/(?:komik/)?[^/?#]+)/?$" - test = ( - ("https://komikcast.site/komik/090-eko-to-issho/", { - "url": "19d3d50d532e84be6280a3d61ff0fd0ca04dd6b4", - "keyword": "837a7e96867344ff59d840771c04c20dc46c0ab1", - }), - ("https://komikcast.me/tonari-no-kashiwagi-san/"), - ) + example = "https://komikcast.site/komik/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/lensdump.py b/gallery_dl/extractor/lensdump.py index 43fc24e1..d4ccf33b 100644 --- a/gallery_dl/extractor/lensdump.py +++ b/gallery_dl/extractor/lensdump.py @@ -48,19 +48,7 @@ class LensdumpBase(): class LensdumpAlbumExtractor(LensdumpBase, GalleryExtractor): subcategory = "album" pattern = BASE_PATTERN + r"/(?:((?!\w+/albums|a/|i/)\w+)|a/(\w+))" - test = ( - ("https://lensdump.com/a/1IhJr", { - "pattern": r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", - "keyword": { - "extension": "png", - "name": str, - "num": int, - "title": str, - "url": str, - "width": int, - }, - }), - ) + example = "https://lensdump.com/a/ID" def __init__(self, match): GalleryExtractor.__init__(self, match, match.string) @@ -100,7 +88,7 @@ class LensdumpAlbumsExtractor(LensdumpBase, Extractor): """Extractor for album list from lensdump.com""" subcategory = "albums" pattern = BASE_PATTERN + r"/\w+/albums" - test = ("https://lensdump.com/vstar925/albums",) + example = "https://lensdump.com/USER/albums" def items(self): for node in self.nodes(): @@ -117,22 +105,7 @@ class LensdumpImageExtractor(LensdumpBase, Extractor): directory_fmt = ("{category}",) archive_fmt = "{id}" pattern = BASE_PATTERN + r"/i/(\w+)" - test = ( - ("https://lensdump.com/i/tyoAyM", { - "pattern": r"https://c\.l3n\.co/i/tyoAyM\.webp", - "content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", - "keyword": { - "date": "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename": "tyoAyM", - "height": 400, - "id": "tyoAyM", - "title": "MYOBI clovis bookcaseset", - "url": "https://c.l3n.co/i/tyoAyM.webp", - "width": 620, - }, - }), - ) + example = "https://lensdump.com/i/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/lexica.py b/gallery_dl/extractor/lexica.py index ad936257..d55d821e 100644 --- a/gallery_dl/extractor/lexica.py +++ b/gallery_dl/extractor/lexica.py @@ -20,37 +20,7 @@ class LexicaSearchExtractor(Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "{id}" pattern = r"(?:https?://)?lexica\.art/?\?q=([^&#]+)" - test = ( - ("https://lexica.art/?q=tree", { - "pattern": r"https://lexica-serve-encoded-images2\.sharif\." - r"workers.dev/full_jpg/[0-9a-f-]{36}$", - "range": "1-80", - "count": 80, - "keyword": { - "height": int, - "id": str, - "upscaled_height": int, - "upscaled_width": int, - "userid": str, - "width": int, - "prompt": { - "c": int, - "grid": bool, - "height": int, - "id": str, - "images": list, - "initImage": None, - "initImageStrength": None, - "model": "lexica-aperture-v2", - "negativePrompt": str, - "prompt": str, - "seed": str, - "timestamp": r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", - "width": int, - }, - }, - }), - ) + example = "https://lexica.art/?q=QUERY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/lightroom.py b/gallery_dl/extractor/lightroom.py index 783473d6..2cbaa976 100644 --- a/gallery_dl/extractor/lightroom.py +++ b/gallery_dl/extractor/lightroom.py @@ -18,24 +18,7 @@ class LightroomGalleryExtractor(Extractor): filename_fmt = "{num:>04}_{id}.{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?lightroom\.adobe\.com/shares/([0-9a-f]+)" - test = ( - (("https://lightroom.adobe.com/shares/" - "0c9cce2033f24d24975423fe616368bf"), { - "keyword": { - "title": "Sterne und Nachtphotos", - "user": "Christian Schrang", - }, - "count": ">= 55", - }), - (("https://lightroom.adobe.com/shares/" - "7ba68ad5a97e48608d2e6c57e6082813"), { - "keyword": { - "title": "HEBFC Snr/Res v Brighton", - "user": "", - }, - "count": ">= 180", - }), - ) + example = "https://lightroom.adobe.com/shares/0123456789abcdef" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/livedoor.py b/gallery_dl/extractor/livedoor.py index 2765f0b6..e21659f0 100644 --- a/gallery_dl/extractor/livedoor.py +++ b/gallery_dl/extractor/livedoor.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -84,32 +84,7 @@ class LivedoorBlogExtractor(LivedoorExtractor): """Extractor for a user's blog on blog.livedoor.jp""" subcategory = "blog" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/?(?:$|[?#])" - test = ( - ("http://blog.livedoor.jp/zatsu_ke/", { - "range": "1-50", - "count": 50, - "archive": False, - "pattern": r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", - "keyword": { - "post": { - "categories" : tuple, - "date" : "type:datetime", - "description": str, - "id" : int, - "tags" : list, - "title" : str, - "user" : "zatsu_ke" - }, - "filename": str, - "hash" : r"re:\w{4,}", - "num" : int, - }, - }), - ("http://blog.livedoor.jp/uotapo/", { - "range": "1-5", - "count": 5, - }), - ) + example = "http://blog.livedoor.jp/USER/" def posts(self): url = "{}/{}".format(self.root, self.user) @@ -129,20 +104,7 @@ class LivedoorPostExtractor(LivedoorExtractor): """Extractor for images from a blog post on blog.livedoor.jp""" subcategory = "post" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/archives/(\d+)" - test = ( - ("http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", { - "url": "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", - "keyword": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", - }), - ("http://blog.livedoor.jp/amaumauma/archives/7835811.html", { - "url": "204bbd6a9db4969c50e0923855aeede04f2e4a62", - "keyword": "05821c7141360e6057ef2d382b046f28326a799d", - }), - ("http://blog.livedoor.jp/uotapo/archives/1050616939.html", { - "url": "4b5ab144b7309eb870d9c08f8853d1abee9946d2", - "keyword": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", - }), - ) + example = "http://blog.livedoor.jp/USER/archives/12345.html" def __init__(self, match): LivedoorExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/lolisafe.py b/gallery_dl/extractor/lolisafe.py index 9cebe3ae..3d7d685b 100644 --- a/gallery_dl/extractor/lolisafe.py +++ b/gallery_dl/extractor/lolisafe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,17 +30,7 @@ BASE_PATTERN = LolisafeExtractor.update({ class LolisafeAlbumExtractor(LolisafeExtractor): subcategory = "album" pattern = BASE_PATTERN + "/a/([^/?#]+)" - test = ( - ("https://xbunkr.com/a/TA0bu3F4", { - "pattern": r"https://media\.xbunkr\.com/[^.]+\.\w+", - "count": 861, - "keyword": { - "album_id": "TA0bu3F4", - "album_name": "Hannahowo Onlyfans Photos", - } - }), - ("https://xbunkr.com/a/GNQc2I5d"), - ) + example = "https://xbunkr.com/a/ID" def __init__(self, match): LolisafeExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/luscious.py b/gallery_dl/extractor/luscious.py index dcf09d16..c3c44d2c 100644 --- a/gallery_dl/extractor/luscious.py +++ b/gallery_dl/extractor/luscious.py @@ -47,73 +47,7 @@ class LusciousAlbumExtractor(LusciousExtractor): archive_fmt = "{album[id]}_{id}" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)") - test = ( - ("https://luscious.net/albums/okinami-no-koigokoro_277031/", { - "pattern": r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+" - r"/images/NTRshouldbeillegal/277031" - r"/luscious_net_\d+_\d+\.jpg$", - # "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3", - "keyword": { - "album": { - "__typename" : "Album", - "audiences" : list, - "content" : "Hentai", - "cover" : "re:https://\\w+.luscious.net/.+/277031/", - "created" : 1479625853, - "created_by" : "NTRshouldbeillegal", - "date" : "dt:2016-11-20 07:10:53", - "description" : "Enjoy.", - "download_url": "re:/download/(r/)?824778/277031/", - "genres" : list, - "id" : 277031, - "is_manga" : True, - "labels" : list, - "language" : "English", - "like_status" : "none", - "modified" : int, - "permissions" : list, - "rating" : float, - "slug" : "okinami-no-koigokoro", - "status" : None, - "tags" : list, - "title" : "Okinami no Koigokoro", - "url" : "/albums/okinami-no-koigokoro_277031/", - "marked_for_deletion": False, - "marked_for_processing": False, - "number_of_animated_pictures": 0, - "number_of_favorites": int, - "number_of_pictures": 18, - }, - "aspect_ratio": r"re:\d+:\d+", - "category" : "luscious", - "created" : int, - "date" : "type:datetime", - "height" : int, - "id" : int, - "is_animated" : False, - "like_status" : "none", - "position" : int, - "resolution" : r"re:\d+x\d+", - "status" : None, - "tags" : list, - "thumbnail" : str, - "title" : str, - "width" : int, - "number_of_comments": int, - "number_of_favorites": int, - }, - }), - ("https://luscious.net/albums/not-found_277035/", { - "exception": exception.NotFoundError, - }), - ("https://members.luscious.net/albums/login-required_323871/", { - "count": 64, - }), - ("https://www.luscious.net/albums/okinami_277031/"), - ("https://members.luscious.net/albums/okinami_277031/"), - ("https://luscious.net/pictures/c/video_game_manga/album" - "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"), - ) + example = "https://luscious.net/albums/TITLE_12345/" def __init__(self, match): LusciousExtractor.__init__(self, match) @@ -338,15 +272,7 @@ class LusciousSearchExtractor(LusciousExtractor): subcategory = "search" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" r"/albums/list/?(?:\?([^#]+))?") - test = ( - ("https://members.luscious.net/albums/list/"), - ("https://members.luscious.net/albums/list/" - "?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", { - "pattern": LusciousAlbumExtractor.pattern, - "range": "41-60", - "count": 20, - }), - ) + example = "https://luscious.net/albums/list/?tagged=TAG" def __init__(self, match): LusciousExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/lynxchan.py b/gallery_dl/extractor/lynxchan.py index 85e8bb1e..0edd5c12 100644 --- a/gallery_dl/extractor/lynxchan.py +++ b/gallery_dl/extractor/lynxchan.py @@ -40,22 +40,7 @@ class LynxchanThreadExtractor(LynxchanExtractor): filename_fmt = "{postId}{num:?-//} {filename}.{extension}" archive_fmt = "{boardUri}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://bbw-chan.nl/bbwdraw/res/499.html", { - "pattern": r"https://bbw-chan\.nl/\.media/[0-9a-f]{64}(\.\w+)?$", - "count": ">= 352", - }), - ("https://bbw-chan.nl/bbwdraw/res/489.html"), - ("https://kohlchan.net/a/res/4594.html", { - "pattern": r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", - "count": ">= 80", - }), - ("https://endchan.org/yuri/res/193483.html", { - "pattern": r"https://endchan\.org/\.media/[^.]+(\.\w+)?$", - "count" : ">= 19", - }), - ("https://endchan.org/yuri/res/33621.html"), - ) + example = "https://bbw-chan.nl/a/res/12345.html" def __init__(self, match): LynxchanExtractor.__init__(self, match) @@ -86,24 +71,7 @@ class LynxchanBoardExtractor(LynxchanExtractor): """Extractor for LynxChan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" - test = ( - ("https://bbw-chan.nl/bbwdraw/", { - "pattern": LynxchanThreadExtractor.pattern, - "count": ">= 148", - }), - ("https://bbw-chan.nl/bbwdraw/2.html"), - ("https://kohlchan.net/a/", { - "pattern": LynxchanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://kohlchan.net/a/2.html"), - ("https://kohlchan.net/a/catalog.html"), - ("https://endchan.org/yuri/", { - "pattern": LynxchanThreadExtractor.pattern, - "count" : ">= 9", - }), - ("https://endchan.org/yuri/catalog.html"), - ) + example = "https://bbw-chan.nl/a/" def __init__(self, match): LynxchanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index e82ac91a..dbaf4cb8 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -98,25 +98,8 @@ class MangadexChapterExtractor(MangadexExtractor): """Extractor for manga-chapters from mangadex.org""" subcategory = "chapter" pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)" - test = ( - ("https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", { - "keyword": "e86128a79ebe7201b648f1caa828496a2878dc8f", - # "content": "50383a4c15124682057b197d40261641a98db514", - }), - # oneshot - ("https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", { - "count": 64, - "keyword": "d11ed057a919854696853362be35fc0ba7dded4c", - }), - # MANGA Plus (#1154) - ("https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", { - "exception": exception.StopExtraction, - }), - # 'externalUrl', but still downloadable / 404 (#2503) - ("https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", { - "count": 0, - }), - ) + example = ("https://mangadex.org/chapter" + "/01234567-89ab-cdef-0123-456789abcdef") def items(self): try: @@ -148,48 +131,8 @@ class MangadexMangaExtractor(MangadexExtractor): """Extractor for manga from mangadex.org""" subcategory = "manga" pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)" - test = ( - ("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", { - "count": ">= 5", - "keyword": { - "manga" : "Souten no Koumori", - "manga_id": "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "title" : "re:One[Ss]hot", - "volume" : 0, - "chapter" : 0, - "chapter_minor": "", - "chapter_id": str, - "date" : "type:datetime", - "lang" : str, - "language": str, - "artist" : ["Arakawa Hiromu"], - "author" : ["Arakawa Hiromu"], - "status" : "completed", - "tags" : ["Oneshot", "Historical", "Action", - "Martial Arts", "Drama", "Tragedy"], - }, - }), - # mutliple values for 'lang' (#4093) - ("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", { - "options": (("lang", "fr,it"),), - "count": 2, - "keyword": { - "manga" : "Souten no Koumori", - "lang" : "re:fr|it", - "language": "re:French|Italian", - }, - }), - ("https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", { - "options": (("lang", "en"),), - "count": ">= 100", - }), - ("https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", { - "count": 1, - }), - ("https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", { - "count": ">= 20", - }) - ) + example = ("https://mangadex.org/title" + "/01234567-89ab-cdef-0123-456789abcdef") def chapters(self): return self.api.manga_feed(self.uuid) @@ -199,7 +142,7 @@ class MangadexFeedExtractor(MangadexExtractor): """Extractor for chapters from your Followed Feed""" subcategory = "feed" pattern = BASE_PATTERN + r"/title/feed$()" - test = ("https://mangadex.org/title/feed",) + example = "https://mangadex.org/title/feed" def chapters(self): return self.api.user_follows_manga_feed() diff --git a/gallery_dl/extractor/mangafox.py b/gallery_dl/extractor/mangafox.py index 8478b8de..2becfa54 100644 --- a/gallery_dl/extractor/mangafox.py +++ b/gallery_dl/extractor/mangafox.py @@ -20,14 +20,7 @@ class MangafoxChapterExtractor(ChapterExtractor): root = "https://m.fanfox.net" pattern = BASE_PATTERN + \ r"(/manga/[^/?#]+/((?:v([^/?#]+)/)?c(\d+)([^/?#]*)))" - test = ( - ("http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", { - "keyword": "5661dab258d42d09d98f194f7172fb9851a49766", - "content": "5c50c252dcf12ffecf68801f4db8a2167265f66c", - }), - ("http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/"), - ("http://fanfox.net/manga/black_clover/vTBD/c295/1.html"), - ) + example = "https://fanfox.net/manga/TITLE/v01/c001/1.html" def __init__(self, match): base, self.cstr, self.volume, self.chapter, self.minor = match.groups() @@ -73,36 +66,7 @@ class MangafoxMangaExtractor(MangaExtractor): root = "https://m.fanfox.net" chapterclass = MangafoxChapterExtractor pattern = BASE_PATTERN + r"(/manga/[^/?#]+)/?$" - test = ( - ("https://fanfox.net/manga/kanojo_mo_kanojo", { - "pattern": MangafoxChapterExtractor.pattern, - "count": ">=60", - "keyword": { - "author": "HIROYUKI", - "chapter": int, - "chapter_minor": r"re:^(\.\d+)?$", - "chapter_string": r"re:(v\d+/)?c\d+", - "date": "type:datetime", - "description": "High school boy Naoya gets a confession from M" - "omi, a cute and friendly girl. However, Naoya " - "already has a girlfriend, Seki... but Momi is " - "too good a catch to let go. Momi and Nagoya's " - "goal becomes clear: convince Seki to accept be" - "ing an item with the two of them. Will she bud" - "ge?", - "lang": "en", - "language": "English", - "manga": "Kanojo mo Kanojo", - "tags": ["Comedy", "Romance", "School Life", "Shounen"], - "volume": int, - }, - }), - ("https://mangafox.me/manga/shangri_la_frontier", { - "pattern": MangafoxChapterExtractor.pattern, - "count": ">=45", - }), - ("https://m.fanfox.net/manga/sentai_daishikkaku"), - ) + example = "https://fanfox.net/manga/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/mangahere.py b/gallery_dl/extractor/mangahere.py index 745231b1..e8ee8612 100644 --- a/gallery_dl/extractor/mangahere.py +++ b/gallery_dl/extractor/mangahere.py @@ -25,18 +25,7 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor): """Extractor for manga-chapters from mangahere.cc""" pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]/manga/" r"([^/]+(?:/v0*(\d+))?/c([^/?#]+))") - test = ( - ("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", { - "keyword": "7c98d7b50a47e6757b089aa875a53aa970cac66f", - "content": "708d475f06893b88549cbd30df1e3f9428f2c884", - }), - # URLs without HTTP scheme (#1070) - ("https://www.mangahere.cc/manga/beastars/c196/1.html", { - "pattern": "https://zjcdn.mangahere.org/.*", - }), - ("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"), - ("http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/"), - ) + example = "https://www.mangahere.cc/manga/TITLE/c001/1.html" def __init__(self, match): self.part, self.volume, self.chapter = match.groups() @@ -95,24 +84,7 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor): chapterclass = MangahereChapterExtractor pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]" r"(/manga/[^/?#]+/?)(?:#.*)?$") - test = ( - ("https://www.mangahere.cc/manga/aria/", { - "url": "9c2e54ec42e9a87ad53096c328b33c90750af3e4", - "keyword": "71503c682c5d0c277a50409a8c5fd78e871e3d69", - "count": 71, - }), - ("https://www.mangahere.cc/manga/hiyokoi/#50", { - "url": "654850570aa03825cd57e2ae2904af489602c523", - "keyword": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", - }), - # adult filter (#556) - ("http://www.mangahere.cc/manga/gunnm_mars_chronicle/", { - "pattern": MangahereChapterExtractor.pattern, - "count": ">= 50", - }), - ("https://www.mangahere.co/manga/aria/"), - ("https://m.mangahere.co/manga/aria/"), - ) + example = "https://www.mangahere.cc/manga/TITLE" def _init(self): self.cookies.set("isAdult", "1", domain="www.mangahere.cc") diff --git a/gallery_dl/extractor/mangakakalot.py b/gallery_dl/extractor/mangakakalot.py index e397586e..2b93392d 100644 --- a/gallery_dl/extractor/mangakakalot.py +++ b/gallery_dl/extractor/mangakakalot.py @@ -25,16 +25,7 @@ class MangakakalotBase(): class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): """Extractor for manga chapters from mangakakalot.tv""" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/chapter[_-][^/?#]+)" - test = ( - ("https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", { - "pattern": r"https://cm\.blazefast\.co" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", - "keyword": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", - "count": 9, - }), - ("https://mangakakalot.tv/chapter" - "/hatarakanai_futari_the_jobless_siblings/chapter_20.1"), - ) + example = "https://ww6.mangakakalot.tv/chapter/manga-ID/chapter-01" def __init__(self, match): self.path = match.group(1) @@ -78,13 +69,7 @@ class MangakakalotMangaExtractor(MangakakalotBase, MangaExtractor): """Extractor for manga from mangakakalot.tv""" chapterclass = MangakakalotChapterExtractor pattern = BASE_PATTERN + r"(/manga/[^/?#]+)" - test = ( - ("https://ww3.mangakakalot.tv/manga/manga-jk986845", { - "pattern": MangakakalotChapterExtractor.pattern, - "count": ">= 30", - }), - ("https://mangakakalot.tv/manga/lk921810"), - ) + example = "https://ww6.mangakakalot.tv/manga/manga-ID" def chapters(self, page): data = {"lang": "en", "language": "English"} diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index 807bc5ee..d2175a02 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -55,27 +55,7 @@ class ManganeloBase(): class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor): """Extractor for manga chapters from manganelo.com""" pattern = BASE_PATTERN + r"(/(?:manga-\w+|chapter/\w+)/chapter[-_][^/?#]+)" - test = ( - ("https://chapmanganato.com/manga-gn983696/chapter-23", { - "pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23" - r"/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", - "keyword": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", - "count": 25, - }), - ("https://chapmanganelo.com/manga-ti107776/chapter-4", { - "pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92" - r"/08/ti970565/chapter_4_caster/\d+-o\.jpg", - "keyword": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", - "count": 45, - }), - ("https://chapmanganato.com/manga-no991297/chapter-8", { - "keyword": {"chapter": 8, "chapter_minor": "-1"}, - "count": 20, - }), - ("https://readmanganato.com/manga-gn983696/chapter-23"), - ("https://manganelo.com/chapter/gamers/chapter_15"), - ("https://manganelo.com/chapter/gq921227/chapter_23"), - ) + example = "https://chapmanganato.com/manga-ID/chapter-01" def metadata(self, page): extr = text.extract_from(page) @@ -104,19 +84,7 @@ class ManganeloMangaExtractor(ManganeloBase, MangaExtractor): """Extractor for manga from manganelo.com""" chapterclass = ManganeloChapterExtractor pattern = BASE_PATTERN + r"(/(?:manga[-/]|read_)\w+)/?$" - test = ( - ("https://chapmanganato.com/manga-gn983696", { - "pattern": ManganeloChapterExtractor.pattern, - "count": ">= 25", - }), - ("https://m.manganelo.com/manga-ti107776", { - "pattern": ManganeloChapterExtractor.pattern, - "count": ">= 12", - }), - ("https://readmanganato.com/manga-gn983696"), - ("https://manganelo.com/manga/read_otome_no_teikoku"), - ("https://manganelo.com/manga/ol921234/"), - ) + example = "https://manganato.com/manga-ID" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index a0d1e80b..6b8f869a 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -35,39 +35,7 @@ class MangaparkBase(): class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor): """Extractor for manga-chapters from mangapark.net""" pattern = BASE_PATTERN + r"/title/[^/?#]+/(\d+)" - test = ( - ("https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", { - "count": 70, - "pattern": r"https://[\w-]+\.mpcdn\.org/comic/2002/e67" - r"/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg" - r"\?acc=[^&#]+&exp=\d+", - "keyword": { - "artist": [], - "author": ["Amano Kozue"], - "chapter": 60, - "chapter_id": 6710214, - "chapter_minor": ".2", - "count": 70, - "date": "dt:2022-01-15 09:25:03", - "extension": "jpeg", - "filename": str, - "genre": ["adventure", "comedy", "drama", "sci_fi", - "shounen", "slice_of_life"], - "lang": "en", - "language": "English", - "manga": "Aria", - "manga_id": 114972, - "page": int, - "source": "Koala", - "title": "Special Navigation - Aquaria Ii", - "volume": 12, - }, - }), - ("https://mangapark.com/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.org/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.io/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.me/title/114972-aria/6710214-en-ch.60.2"), - ) + example = "https://mangapark.net/title/MANGA/12345-en-ch.01" def __init__(self, match): self.root = text.root_from_url(match.group(0)) @@ -115,41 +83,7 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor): """Extractor for manga from mangapark.net""" subcategory = "manga" pattern = BASE_PATTERN + r"/title/(\d+)(?:-[^/?#]*)?/?$" - test = ( - ("https://mangapark.net/title/114972-aria", { - "count": 141, - "pattern": MangaparkChapterExtractor.pattern, - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": str, - "date": "type:datetime", - "lang": "en", - "language": "English", - "manga_id": 114972, - "source": "re:Horse|Koala", - "source_id": int, - "title": str, - "volume": int, - }, - }), - # 'source' option - ("https://mangapark.net/title/114972-aria", { - "options": (("source", "koala"),), - "count": 70, - "pattern": MangaparkChapterExtractor.pattern, - "keyword": { - "source": "Koala", - "source_id": 15150116, - }, - }), - ("https://mangapark.com/title/114972-"), - ("https://mangapark.com/title/114972"), - ("https://mangapark.com/title/114972-aria"), - ("https://mangapark.org/title/114972-aria"), - ("https://mangapark.io/title/114972-aria"), - ("https://mangapark.me/title/114972-aria"), - ) + example = "https://mangapark.net/title/MANGA" def __init__(self, match): self.root = text.root_from_url(match.group(0)) diff --git a/gallery_dl/extractor/mangaread.py b/gallery_dl/extractor/mangaread.py index 74c239e9..8f193741 100644 --- a/gallery_dl/extractor/mangaread.py +++ b/gallery_dl/extractor/mangaread.py @@ -35,56 +35,7 @@ class MangareadChapterExtractor(MangareadBase, ChapterExtractor): """Extractor for manga-chapters from mangaread.org""" pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org" r"(/manga/[^/?#]+/[^/?#]+)") - test = ( - ("https://www.mangaread.org/manga/one-piece/chapter-1053-3/", { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 11, - "keyword": { - "manga" : "One Piece", - "title" : "", - "chapter" : 1053, - "chapter_minor": ".3", - "tags" : ["Oda Eiichiro"], - "lang" : "en", - "language": "English", - } - }), - ("https://www.mangaread.org/manga/one-piece/chapter-1000000/", { - "exception": exception.NotFoundError, - }), - (("https://www.mangaread.org" - "/manga/kanan-sama-wa-akumade-choroi/chapter-10/"), { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 9, - "keyword": { - "manga" : "Kanan-sama wa Akumade Choroi", - "title" : "", - "chapter" : 10, - "chapter_minor": "", - "tags" : list, - "lang" : "en", - "language": "English", - } - }), - # 'Chapter146.5' - # ^^ no whitespace - ("https://www.mangaread.org/manga/above-all-gods/chapter146-5/", { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 6, - "keyword": { - "manga" : "Above All Gods", - "title" : "", - "chapter" : 146, - "chapter_minor": ".5", - "tags" : list, - "lang" : "en", - "language": "English", - } - }), - ) + example = "https://www.mangaread.org/manga/MANGA/chapter-01/" def metadata(self, page): tags = text.extr(page, 'class="wp-manga-tags-list">', '</div>') @@ -108,50 +59,7 @@ class MangareadMangaExtractor(MangareadBase, MangaExtractor): """Extractor for manga from mangaread.org""" chapterclass = MangareadChapterExtractor pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$" - test = ( - ("https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", { - "pattern": (r"https://www\.mangaread\.org/manga" - r"/kanan-sama-wa-akumade-choroi" - r"/chapter-\d+(-.+)?/"), - "count" : ">= 13", - "keyword": { - "manga" : "Kanan-sama wa Akumade Choroi", - "author" : ["nonco"], - "artist" : ["nonco"], - "type" : "Manga", - "genres" : ["Comedy", "Romance", "Shounen", "Supernatural"], - "rating" : float, - "release": 2022, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : list, - "description": str, - } - }), - ("https://www.mangaread.org/manga/one-piece", { - "pattern": (r"https://www\.mangaread\.org/manga" - r"/one-piece/chapter-\d+(-.+)?/"), - "count" : ">= 1066", - "keyword": { - "manga" : "One Piece", - "author" : ["Oda Eiichiro"], - "artist" : ["Oda Eiichiro"], - "type" : "Manga", - "genres" : list, - "rating" : float, - "release": 1997, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : ["One Piece"], - "description": str, - } - }), - ("https://www.mangaread.org/manga/doesnotexist", { - "exception": exception.HttpError, - }), - ) + example = "https://www.mangaread.org/manga/MANGA" def chapters(self, page): if 'class="error404' in page: diff --git a/gallery_dl/extractor/mangasee.py b/gallery_dl/extractor/mangasee.py index 00c89c1e..72613325 100644 --- a/gallery_dl/extractor/mangasee.py +++ b/gallery_dl/extractor/mangasee.py @@ -36,54 +36,7 @@ class MangaseeBase(): class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor): pattern = (r"(?:https?://)?(mangasee123|manga4life)\.com" r"(/read-online/[^/?#]+\.html)") - test = ( - (("https://mangasee123.com/read-online" - "/Tokyo-Innocent-chapter-4.5-page-1.html"), { - "pattern": r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", - "count": 8, - "keyword": { - "author": ["NARUMI Naru"], - "chapter": 4, - "chapter_minor": ".5", - "chapter_string": "100045", - "count": 8, - "date": "dt:2020-01-20 21:52:53", - "extension": "png", - "filename": r"re:0004\.5-00\d", - "genre": ["Comedy", "Fantasy", "Harem", "Romance", "Shounen", - "Supernatural"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Tokyo Innocent", - "page": int, - "title": "", - }, - }), - (("https://manga4life.com/read-online" - "/One-Piece-chapter-1063-page-1.html"), { - "pattern": r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", - "count": 13, - "keyword": { - "author": ["ODA Eiichiro"], - "chapter": 1063, - "chapter_minor": "", - "chapter_string": "110630", - "count": 13, - "date": "dt:2022-10-16 17:32:54", - "extension": "png", - "filename": r"re:1063-0\d\d", - "genre": ["Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Shounen"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "One Piece", - "page": int, - "title": "", - }, - }), - ) + example = "https://mangasee123.com/read-online/MANGA-chapter-1-page-1.html" def __init__(self, match): if match.group(1) == "manga4life": @@ -134,45 +87,7 @@ class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor): class MangaseeMangaExtractor(MangaseeBase, MangaExtractor): chapterclass = MangaseeChapterExtractor pattern = r"(?:https?://)?(mangasee123|manga4life)\.com(/manga/[^/?#]+)" - test = ( - (("https://mangasee123.com/manga" - "/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai"), { - "pattern": MangaseeChapterExtractor.pattern, - "count": ">= 17", - "keyword": { - "author": ["TAKASE Masaya"], - "chapter": int, - "chapter_minor": r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date": "type:datetime", - "genre": ["Comedy", "Romance", "School Life", "Shounen", - "Slice of Life"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "title": "", - }, - }), - ("https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", { - "pattern": MangaseeChapterExtractor.pattern, - "count": ">= 50", - "keyword": { - "author": ["Canno"], - "chapter": int, - "chapter_minor": r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date": "type:datetime", - "genre": ["Comedy", "Romance", "School Life", "Seinen", - "Shoujo Ai"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "title": "" - }, - }), - ) + example = "https://mangasee123.com/manga/MANGA" def __init__(self, match): if match.group(1) == "manga4life": diff --git a/gallery_dl/extractor/mangoxo.py b/gallery_dl/extractor/mangoxo.py index cca18b13..b208f035 100644 --- a/gallery_dl/extractor/mangoxo.py +++ b/gallery_dl/extractor/mangoxo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -80,25 +80,7 @@ class MangoxoAlbumExtractor(MangoxoExtractor): directory_fmt = ("{category}", "{channel[name]}", "{album[name]}") archive_fmt = "{album[id]}_{num}" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/album/(\w+)" - test = ("https://www.mangoxo.com/album/lzVOv1Q9", { - "url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", - "keyword": { - "channel": { - "id": "gaxO16d8", - "name": "Phoenix", - "cover": str, - }, - "album": { - "id": "lzVOv1Q9", - "name": "re:池永康晟 Ikenaga Yasunari 透出古朴", - "date": "dt:2019-03-22 14:42:00", - "description": str, - }, - "id": int, - "num": int, - "count": 65, - }, - }) + example = "https://www.mangoxo.com/album/ID" def __init__(self, match): MangoxoExtractor.__init__(self, match) @@ -163,11 +145,7 @@ class MangoxoChannelExtractor(MangoxoExtractor): """Extractor for all albums on a mangoxo channel""" subcategory = "channel" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/(\w+)/album" - test = ("https://www.mangoxo.com/phoenix/album", { - "pattern": MangoxoAlbumExtractor.pattern, - "range": "1-30", - "count": "> 20", - }) + example = "https://www.mangoxo.com/USER/album" def __init__(self, match): MangoxoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 3bed955c..cb94481e 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -106,29 +106,7 @@ class MastodonUserExtractor(MastodonExtractor): """Extractor for all images of an account/user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)(?:/media)?/?$" - test = ( - ("https://mastodon.social/@jk", { - "pattern": r"https://files.mastodon.social/media_attachments" - r"/files/(\d+/){3,}original/\w+", - "range": "1-60", - "count": 60, - }), - ("https://pawoo.net/@yoru_nine/", { - "range": "1-60", - "count": 60, - }), - ("https://baraag.net/@pumpkinnsfw"), - ("https://mastodon.social/@yoru_nine@pawoo.net", { - "pattern": r"https://mastodon\.social/media_proxy/\d+/original", - "range": "1-10", - "count": 10, - }), - ("https://mastodon.social/@id:10843"), - ("https://mastodon.social/users/id:10843"), - ("https://mastodon.social/users/jk"), - ("https://mastodon.social/users/yoru_nine@pawoo.net"), - ("https://mastodon.social/web/@jk"), - ) + example = "https://mastodon.social/@USER" def statuses(self): api = MastodonAPI(self) @@ -144,11 +122,7 @@ class MastodonBookmarkExtractor(MastodonExtractor): """Extractor for mastodon bookmarks""" subcategory = "bookmark" pattern = BASE_PATTERN + r"/bookmarks" - test = ( - ("https://mastodon.social/bookmarks"), - ("https://pawoo.net/bookmarks"), - ("https://baraag.net/bookmarks"), - ) + example = "https://mastodon.social/bookmarks" def statuses(self): return MastodonAPI(self).account_bookmarks() @@ -158,15 +132,7 @@ class MastodonFollowingExtractor(MastodonExtractor): """Extractor for followed mastodon users""" subcategory = "following" pattern = BASE_PATTERN + r"/users/([^/?#]+)/following" - test = ( - ("https://mastodon.social/users/0x4f/following", { - "extractor": False, - "count": ">= 20", - }), - ("https://mastodon.social/users/id:10843/following"), - ("https://pawoo.net/users/yoru_nine/following"), - ("https://baraag.net/users/pumpkinnsfw/following"), - ) + example = "https://mastodon.social/users/USER/following" def items(self): api = MastodonAPI(self) @@ -181,21 +147,7 @@ class MastodonStatusExtractor(MastodonExtractor): """Extractor for images from a status""" subcategory = "status" pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)" - test = ( - ("https://mastodon.social/@jk/103794036899778366", { - "count": 4, - "keyword": { - "count": 4, - "num": int, - }, - }), - ("https://pawoo.net/@yoru_nine/105038878897832922", { - "content": "b52e807f8ab548d6f896b09218ece01eba83987a", - }), - ("https://baraag.net/@pumpkinnsfw/104364170556898443", { - "content": "67748c1b828c58ad60d0fe5729b59fb29c872244", - }), - ) + example = "https://mastodon.social/@USER/12345" def statuses(self): return (MastodonAPI(self).status(self.item),) diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 8c717581..95b83b62 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -85,24 +85,7 @@ class MisskeyUserExtractor(MisskeyExtractor): """Extractor for all images of a Misskey user""" subcategory = "user" pattern = BASE_PATTERN + r"/@([^/?#]+)/?$" - test = ( - ("https://misskey.io/@lithla", { - "pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", - "range": "1-50", - "count": 50, - }), - ("https://misskey.io/@blooddj@pawoo.net", { - "range": "1-50", - "count": 50, - }), - ("https://lesbian.energy/@rerorero", { - "pattern": r"https://lesbian.energy/files/\w+", - "range": "1-50", - "count": 50, - }), - ("https://lesbian.energy/@nano@mk.yopo.work"), - ("https://sushi.ski/@ui@misskey.04.si"), - ) + example = "https://misskey.io/@USER" def notes(self): return self.api.users_notes(self.api.user_id_by_username(self.item)) @@ -112,13 +95,7 @@ class MisskeyFollowingExtractor(MisskeyExtractor): """Extractor for followed Misskey users""" subcategory = "following" pattern = BASE_PATTERN + r"/@([^/?#]+)/following" - test = ( - ("https://misskey.io/@blooddj@pawoo.net/following", { - "extractor": False, - "count": ">= 6", - }), - ("https://sushi.ski/@hatusimo_sigure/following"), - ) + example = "https://misskey.io/@USER/following" def items(self): user_id = self.api.user_id_by_username(self.item) @@ -136,21 +113,7 @@ class MisskeyNoteExtractor(MisskeyExtractor): """Extractor for images from a Note""" subcategory = "note" pattern = BASE_PATTERN + r"/notes/(\w+)" - test = ( - ("https://misskey.io/notes/9bhqfo835v", { - "pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", - "count": 4, - }), - ("https://misskey.io/notes/9brq7z1re6"), - ("https://sushi.ski/notes/9bm3x4ksqw", { - "pattern": r"https://media\.sushi\.ski/files/[\w-]+\.png", - "count": 1, - }), - ("https://lesbian.energy/notes/995ig09wqy", { - "count": 1, - }), - ("https://lesbian.energy/notes/96ynd9w5kc"), - ) + example = "https://misskey.io/notes/98765" def notes(self): return (self.api.notes_show(self.item),) @@ -160,12 +123,7 @@ class MisskeyFavoriteExtractor(MisskeyExtractor): """Extractor for favorited notes""" subcategory = "favorite" pattern = BASE_PATTERN + r"/(?:my|api/i)/favorites" - test = ( - ("https://misskey.io/my/favorites"), - ("https://misskey.io/api/i/favorites"), - ("https://lesbian.energy/my/favorites"), - ("https://sushi.ski/my/favorites"), - ) + example = "https://misskey.io/my/favorites" def notes(self): return self.api.i_favorites() diff --git a/gallery_dl/extractor/moebooru.py b/gallery_dl/extractor/moebooru.py index 1e56bde9..145dd51d 100644 --- a/gallery_dl/extractor/moebooru.py +++ b/gallery_dl/extractor/moebooru.py @@ -92,94 +92,12 @@ BASE_PATTERN = MoebooruExtractor.update({ }) -class MoebooruPostExtractor(MoebooruExtractor): - subcategory = "post" - archive_fmt = "{id}" - pattern = BASE_PATTERN + r"/post/show/(\d+)" - test = ( - ("https://yande.re/post/show/51824", { - "content": "59201811c728096b2d95ce6896fd0009235fe683", - "options": (("tags", True),), - "keyword": { - "tags_artist": "sasaki_tamaru", - "tags_circle": "softhouse_chara", - "tags_copyright": "ouzoku", - "tags_general": str, - }, - }), - ("https://konachan.com/post/show/205189", { - "content": "674e75a753df82f5ad80803f575818b8e46e4b65", - "options": (("tags", True),), - "keyword": { - "tags_artist": "patata", - "tags_character": "clownpiece", - "tags_copyright": "touhou", - "tags_general": str, - }, - }), - ("https://yande.re/post/show/993156", { - "content": "fed722bd90f48de41ec163692befc701056e2b1e", - "options": (("notes", True),), - "keyword": { - "notes": [ - { - "id": 7096, - "x" : 90, - "y" : 626, - "width" : 283, - "height": 529, - "body" : "Please keep this as a secret for me!!", - }, - { - "id": 7095, - "x" : 900, - "y" : 438, - "width" : 314, - "height": 588, - "body" : "The facts that I love playing games", - }, - ], - }, - }), - ("https://lolibooru.moe/post/show/281305/", { - "content": "a331430223ffc5b23c31649102e7d49f52489b57", - "options": (("notes", True),), - "keyword": { - "notes": list, - }, - }), - ("https://konachan.net/post/show/205189"), - ("https://www.sakugabooru.com/post/show/125570"), - ("https://lolibooru.moe/post/show/287835"), - ) - - def __init__(self, match): - MoebooruExtractor.__init__(self, match) - self.post_id = match.group(match.lastindex) - - def posts(self): - params = {"tags": "id:" + self.post_id} - return self.request(self.root + "/post.json", params=params).json() - - class MoebooruTagExtractor(MoebooruExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/post\?(?:[^&#]*&)*tags=([^&#]*)" - test = ( - ("https://yande.re/post?tags=ouzoku+armor", { - "content": "59201811c728096b2d95ce6896fd0009235fe683", - }), - ("https://konachan.com/post?tags=patata", { - "content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", - }), - # empty 'tags' (#4354) - ("https://konachan.com/post?tags="), - ("https://konachan.net/post?tags=patata"), - ("https://www.sakugabooru.com/post?tags=nichijou"), - ("https://lolibooru.moe/post?tags=ruu_%28tksymkw%29"), - ) + example = "https://yande.re/post?tags=TAG" def __init__(self, match): MoebooruExtractor.__init__(self, match) @@ -199,17 +117,7 @@ class MoebooruPoolExtractor(MoebooruExtractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/pool/show/(\d+)" - test = ( - ("https://yande.re/pool/show/318", { - "content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", - }), - ("https://konachan.com/pool/show/95", { - "content": "cf0546e38a93c2c510a478f8744e60687b7a8426", - }), - ("https://konachan.net/pool/show/95"), - ("https://www.sakugabooru.com/pool/show/54"), - ("https://lolibooru.moe/pool/show/239"), - ) + example = "https://yande.re/pool/show/12345" def __init__(self, match): MoebooruExtractor.__init__(self, match) @@ -223,25 +131,28 @@ class MoebooruPoolExtractor(MoebooruExtractor): return self._pagination(self.root + "/post.json", params) +class MoebooruPostExtractor(MoebooruExtractor): + subcategory = "post" + archive_fmt = "{id}" + pattern = BASE_PATTERN + r"/post/show/(\d+)" + example = "https://yande.re/post/show/12345" + + def __init__(self, match): + MoebooruExtractor.__init__(self, match) + self.post_id = match.group(match.lastindex) + + def posts(self): + params = {"tags": "id:" + self.post_id} + return self.request(self.root + "/post.json", params=params).json() + + class MoebooruPopularExtractor(MoebooruExtractor): subcategory = "popular" directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" pattern = BASE_PATTERN + \ r"/post/popular_(by_(?:day|week|month)|recent)(?:\?([^#]*))?" - test = ( - ("https://yande.re/post/popular_by_month?month=6&year=2014", { - "count": 40, - }), - ("https://yande.re/post/popular_recent"), - ("https://konachan.com/post/popular_by_month?month=11&year=2010", { - "count": 20, - }), - ("https://konachan.com/post/popular_recent"), - ("https://konachan.net/post/popular_recent"), - ("https://www.sakugabooru.com/post/popular_recent"), - ("https://lolibooru.moe/post/popular_recent"), - ) + example = "https://yande.re/post/popular_by_month?year=YYYY&month=MM" def __init__(self, match): MoebooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/myhentaigallery.py b/gallery_dl/extractor/myhentaigallery.py index 3301da97..33a22849 100644 --- a/gallery_dl/extractor/myhentaigallery.py +++ b/gallery_dl/extractor/myhentaigallery.py @@ -13,26 +13,11 @@ from .. import text, exception class MyhentaigalleryGalleryExtractor(GalleryExtractor): """Extractor for image galleries from myhentaigallery.com""" category = "myhentaigallery" + root = "https://myhentaigallery.com" directory_fmt = ("{category}", "{gallery_id} {artist:?[/] /J, }{title}") pattern = (r"(?:https?://)?myhentaigallery\.com" r"/gallery/(?:thumbnails|show)/(\d+)") - test = ( - ("https://myhentaigallery.com/gallery/thumbnails/16247", { - "pattern": r"https://images.myhentaicomics\.com/imagesgallery" - r"/images/[^/]+/original/\d+\.jpg", - "keyword": { - "artist" : list, - "count" : 11, - "gallery_id": 16247, - "group" : list, - "parodies" : list, - "tags" : ["Giantess"], - "title" : "Attack Of The 50ft Woman 1", - }, - }), - ("https://myhentaigallery.com/gallery/show/16247/1"), - ) - root = "https://myhentaigallery.com" + example = "https://myhentaigallery.com/gallery/thumbnails/12345" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/myportfolio.py b/gallery_dl/extractor/myportfolio.py index fd16f247..7620d088 100644 --- a/gallery_dl/extractor/myportfolio.py +++ b/gallery_dl/extractor/myportfolio.py @@ -22,27 +22,7 @@ class MyportfolioGalleryExtractor(Extractor): pattern = (r"(?:myportfolio:(?:https?://)?([^/]+)|" r"(?:https?://)?([\w-]+\.myportfolio\.com))" r"(/[^/?#]+)?") - test = ( - ("https://andrewling.myportfolio.com/volvo-xc-90-hybrid", { - "url": "acea0690c76db0e5cf267648cefd86e921bc3499", - "keyword": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", - }), - ("https://andrewling.myportfolio.com/", { - "pattern": r"https://andrewling\.myportfolio\.com/[^/?#+]+$", - "count": ">= 6", - }), - ("https://stevenilousphotography.myportfolio.com/society", { - "exception": exception.NotFoundError, - }), - # custom domain - ("myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", { - "count": 3, - }), - ("myportfolio:https://tooco.com.ar/", { - "pattern": pattern, - "count": ">= 40", - }), - ) + example = "https://USER.myportfolio.com/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/naver.py b/gallery_dl/extractor/naver.py index 7e94b1cc..55faf9e7 100644 --- a/gallery_dl/extractor/naver.py +++ b/gallery_dl/extractor/naver.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -27,17 +27,7 @@ class NaverPostExtractor(NaverBase, GalleryExtractor): archive_fmt = "{blog[id]}_{post[num]}_{num}" pattern = (r"(?:https?://)?blog\.naver\.com/" r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)") - test = ( - ("https://blog.naver.com/rlfqjxm0/221430673006", { - "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", - }), - (("https://blog.naver.com/PostView.nhn" - "?blogId=rlfqjxm0&logNo=221430673006"), { - "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", - }), - ) + example = "https://blog.naver.com/BLOGID/12345" def __init__(self, match): blog_id = match.group(1) @@ -84,18 +74,7 @@ class NaverBlogExtractor(NaverBase, Extractor): categorytransfer = True pattern = (r"(?:https?://)?blog\.naver\.com/" r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)") - test = ( - ("https://blog.naver.com/gukjung", { - "pattern": NaverPostExtractor.pattern, - "count": 12, - "range": "1-12", - }), - ("https://blog.naver.com/PostList.nhn?blogId=gukjung", { - "pattern": NaverPostExtractor.pattern, - "count": 12, - "range": "1-12", - }), - ) + example = "https://blog.naver.com/BLOGID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/naverwebtoon.py b/gallery_dl/extractor/naverwebtoon.py index cafe4f7f..c58056ec 100644 --- a/gallery_dl/extractor/naverwebtoon.py +++ b/gallery_dl/extractor/naverwebtoon.py @@ -28,54 +28,7 @@ class NaverwebtoonEpisodeExtractor(NaverwebtoonBase, GalleryExtractor): filename_fmt = "{episode:>03}-{num:>02}.{extension}" archive_fmt = "{title_id}_{episode}_{num}" pattern = BASE_PATTERN + r"/detail(?:\.nhn)?\?([^#]+)" - test = ( - (("https://comic.naver.com/webtoon/detail" - "?titleId=26458&no=1&weekday=tue"), { - "url": "47a956ba8c7a837213d5985f50c569fcff986f75", - "content": "3806b6e8befbb1920048de9888dfce6220f69a60", - "count": 14, - "keyword": { - "author": ["김규삼"], - "artist": ["김규삼"], - "comic": "N의등대-눈의등대", - "count": 14, - "episode": "1", - "extension": "jpg", - "num": int, - "tags": ["스릴러", "완결무료", "완결스릴러"], - "title": "n의 등대 - 눈의 등대 1화", - "title_id": "26458", - }, - }), - (("https://comic.naver.com/challenge/detail" - "?titleId=765124&no=1"), { - "pattern": r"https://image-comic\.pstatic\.net" - r"/user_contents_data/challenge_comic/2021/01/19" - r"/342586/upload_7149856273586337846\.jpeg", - "count": 1, - "keyword": { - "author": ["kemi****"], - "artist": [], - "comic": "우니 모두의 이야기", - "count": 1, - "episode": "1", - "extension": "jpeg", - "filename": "upload_7149856273586337846", - "num": 1, - "tags": ["일상툰", "우니모두의이야기", "퇴사", "입사", "신입사원", - "사회초년생", "회사원", "20대"], - "title": "퇴사하다", - "title_id": "765124", - }, - }), - (("https://comic.naver.com/bestChallenge/detail.nhn" - "?titleId=771467&no=3"), { - "pattern": r"https://image-comic\.pstatic\.net" - r"/user_contents_data/challenge_comic/2021/04/28" - r"/345534/upload_3617293622396203109\.jpeg", - "count": 1, - }), - ) + example = "https://comic.naver.com/webtoon/detail?titleId=12345&no=1" def __init__(self, match): path, query = match.groups() @@ -115,20 +68,7 @@ class NaverwebtoonComicExtractor(NaverwebtoonBase, Extractor): subcategory = "comic" categorytransfer = True pattern = BASE_PATTERN + r"/list(?:\.nhn)?\?([^#]+)" - test = ( - ("https://comic.naver.com/webtoon/list?titleId=22073", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": 32, - }), - ("https://comic.naver.com/challenge/list?titleId=765124", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": 25, - }), - ("https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": ">= 12", - }), - ) + example = "https://comic.naver.com/webtoon/list?titleId=12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 8a255280..18c1e5c3 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -300,41 +300,7 @@ class NewgroundsImageExtractor(NewgroundsExtractor): pattern = (r"(?:https?://)?(?:" r"(?:www\.)?newgrounds\.com/art/view/([^/?#]+)/[^/?#]+" r"|art\.ngfiles\.com/images/\d+/\d+_([^_]+)_([^.]+))") - test = ( - ("https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", { - "url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", - "content": "8f395e08333eb2457ba8d8b715238f8910221365", - "keyword": { - "artist" : ["tomfulp"], - "comment" : "re:Consider this the bottom threshold for ", - "date" : "dt:2009-06-04 14:44:05", - "description": "re:Consider this the bottom threshold for ", - "favorites" : int, - "filename" : "94_tomfulp_ryu-is-hawt", - "height" : 476, - "index" : 94, - "rating" : "e", - "score" : float, - "tags" : ["ryu", "streetfighter"], - "title" : "Ryu is Hawt", - "type" : "article", - "user" : "tomfulp", - "width" : 447, - }, - }), - ("https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", { - "url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", - }), - ("https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", { - "url": "84eec95e663041a80630df72719f231e157e5f5d", - "count": 2, - }), - # "adult" rated (#2456) - ("https://www.newgrounds.com/art/view/kekiiro/red", { - "options": (("username", None),), - "count": 1, - }), - ) + example = "https://www.newgrounds.com/art/view/USER/TITLE" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -354,104 +320,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor): subcategory = "media" pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" r"(/(?:portal/view|audio/listen)/\d+)") - test = ( - ("https://www.newgrounds.com/portal/view/595355", { - "pattern": r"https://uploads\.ungrounded\.net/alternate/564000" - r"/564957_alternate_31\.mp4\?1359712249", - "keyword": { - "artist" : ["kickinthehead", "danpaladin", "tomfulp"], - "comment" : "re:My fan trailer for Alien Hominid HD!", - "date" : "dt:2013-02-01 09:50:49", - "description": "Fan trailer for Alien Hominid HD!", - "favorites" : int, - "filename" : "564957_alternate_31", - "index" : 595355, - "rating" : "e", - "score" : float, - "tags" : ["alienhominid", "trailer"], - "title" : "Alien Hominid Fan Trailer", - "type" : "movie", - "user" : "kickinthehead", - }, - }), - ("https://www.newgrounds.com/audio/listen/609768", { - "url": "f4c5490ae559a3b05e46821bb7ee834f93a43c95", - "keyword": { - "artist" : ["zj", "tomfulp"], - "comment" : "re:RECORDED 12-09-2014\n\nFrom The ZJ \"Late ", - "date" : "dt:2015-02-23 19:31:59", - "description": "From The ZJ Report Show!", - "favorites" : int, - "index" : 609768, - "rating" : "", - "score" : float, - "tags" : ["fulp", "interview", "tom", "zj"], - "title" : "ZJ Interviews Tom Fulp!", - "type" : "music.song", - "user" : "zj", - }, - }), - # flash animation (#1257) - ("https://www.newgrounds.com/portal/view/161181/format/flash", { - "pattern": r"https://uploads\.ungrounded\.net/161000" - r"/161181_ddautta_mask__550x281_\.swf\?f1081628129", - "keyword": {"type": "movie"}, - }), - # format selection (#1729) - ("https://www.newgrounds.com/portal/view/758545", { - "options": (("format", "720p"),), - "pattern": r"https://uploads\.ungrounded\.net/alternate/1482000" - r"/1482860_alternate_102516\.720p\.mp4\?\d+", - }), - # "adult" rated (#2456) - ("https://www.newgrounds.com/portal/view/717744", { - "options": (("username", None),), - "count": 1, - }), - # flash game - ("https://www.newgrounds.com/portal/view/829032", { - "pattern": r"https://uploads\.ungrounded\.net/829000" - r"/829032_picovsbeardx\.swf\?f1641968445", - "range": "1", - "keyword": { - "artist" : [ - "dungeonation", - "carpetbakery", - "animalspeakandrews", - "bill", - "chipollo", - "dylz49", - "gappyshamp", - "pinktophat", - "rad", - "shapeshiftingblob", - "tomfulp", - "voicesbycorey", - "psychogoldfish", - ], - "comment" : "re:The children are expendable. Take out the ", - "date" : "dt:2022-01-10 23:00:57", - "description": "Bloodshed in The Big House that Blew...again!", - "favorites" : int, - "index" : 829032, - "post_url" : "https://www.newgrounds.com/portal/view/829032", - "rating" : "m", - "score" : float, - "tags" : [ - "assassin", - "boyfriend", - "darnell", - "nene", - "pico", - "picos-school", - ], - "title" : "PICO VS BEAR DX", - "type" : "game", - "url" : "https://uploads.ungrounded.net/829000" - "/829032_picovsbeardx.swf?f1641968445", - }, - }), - ) + example = "https://www.newgrounds.com/portal/view/12345" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -466,58 +335,35 @@ class NewgroundsArtExtractor(NewgroundsExtractor): """Extractor for all images of a newgrounds user""" subcategory = _path = "art" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/art/?$" - test = ("https://tomfulp.newgrounds.com/art", { - "pattern": NewgroundsImageExtractor.pattern, - "count": ">= 3", - }) + example = "https://USER.newgrounds.com/art" class NewgroundsAudioExtractor(NewgroundsExtractor): """Extractor for all audio submissions of a newgrounds user""" subcategory = _path = "audio" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/audio/?$" - test = ("https://tomfulp.newgrounds.com/audio", { - "pattern": r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3", - "count": ">= 4", - }) + example = "https://USER.newgrounds.com/audio" class NewgroundsMoviesExtractor(NewgroundsExtractor): """Extractor for all movies of a newgrounds user""" subcategory = _path = "movies" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$" - test = ("https://tomfulp.newgrounds.com/movies", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "range": "1-10", - "count": 10, - }) + example = "https://USER.newgrounds.com/movies<" class NewgroundsGamesExtractor(NewgroundsExtractor): """Extractor for a newgrounds user's games""" subcategory = _path = "games" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/games/?$" - test = ("https://tomfulp.newgrounds.com/games", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "range": "1-10", - "count": 10, - }) + example = "https://USER.newgrounds.com/games" class NewgroundsUserExtractor(NewgroundsExtractor): """Extractor for a newgrounds user profile""" subcategory = "user" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/?$" - test = ( - ("https://tomfulp.newgrounds.com", { - "pattern": "https://tomfulp.newgrounds.com/art$", - }), - ("https://tomfulp.newgrounds.com", { - "options": (("include", "all"),), - "pattern": "https://tomfulp.newgrounds.com/(art|audio|movies)$", - "count": 3, - }), - ) + example = "https://USER.newgrounds.com" def initialize(self): pass @@ -538,15 +384,7 @@ class NewgroundsFavoriteExtractor(NewgroundsExtractor): directory_fmt = ("{category}", "{user}", "Favorites") pattern = (r"(?:https?://)?([\w-]+)\.newgrounds\.com" r"/favorites(?!/following)(?:/(art|audio|movies))?/?") - test = ( - ("https://tomfulp.newgrounds.com/favorites/art", { - "range": "1-10", - "count": ">= 10", - }), - ("https://tomfulp.newgrounds.com/favorites/audio"), - ("https://tomfulp.newgrounds.com/favorites/movies"), - ("https://tomfulp.newgrounds.com/favorites/"), - ) + example = "https://USER.newgrounds.com/favorites" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -595,11 +433,7 @@ class NewgroundsFollowingExtractor(NewgroundsFavoriteExtractor): """Extractor for a newgrounds user's favorited users""" subcategory = "following" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/favorites/(following)" - test = ("https://tomfulp.newgrounds.com/favorites/following", { - "pattern": NewgroundsUserExtractor.pattern, - "range": "76-125", - "count": 50, - }) + example = "https://USER.newgrounds.com/favorites/following" def items(self): data = {"_extractor": NewgroundsUserExtractor} @@ -620,21 +454,7 @@ class NewgroundsSearchExtractor(NewgroundsExtractor): directory_fmt = ("{category}", "search", "{search_tags}") pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" r"/search/conduct/([^/?#]+)/?\?([^#]+)") - test = ( - ("https://www.newgrounds.com/search/conduct/art?terms=tree", { - "pattern": NewgroundsImageExtractor.pattern, - "keyword": {"search_tags": "tree"}, - "range": "1-10", - "count": 10, - }), - ("https://www.newgrounds.com/search/conduct/movies?terms=tree", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", - "range": "1-10", - "count": 10, - }), - ("https://www.newgrounds.com/search/conduct/audio?advanced=1" - "&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm"), - ) + example = "https://www.newgrounds.com/search/conduct/art?terms=QUERY" def __init__(self, match): NewgroundsExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/nhentai.py b/gallery_dl/extractor/nhentai.py index 4270c84a..09b2b167 100644 --- a/gallery_dl/extractor/nhentai.py +++ b/gallery_dl/extractor/nhentai.py @@ -18,29 +18,7 @@ class NhentaiGalleryExtractor(GalleryExtractor): category = "nhentai" root = "https://nhentai.net" pattern = r"(?:https?://)?nhentai\.net/g/(\d+)" - test = ("https://nhentai.net/g/147850/", { - "url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", - "keyword": { - "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", - "title_en" : str, - "title_ja" : str, - "gallery_id": 147850, - "media_id" : 867789, - "count" : 16, - "date" : 1446050915, - "scanlator" : "", - "artist" : ["morris"], - "group" : list, - "parody" : list, - "characters": list, - "tags" : list, - "type" : "manga", - "lang" : "en", - "language" : "English", - "width" : int, - "height" : int, - }, - }) + example = "https://nhentai.net/g/12345/" def __init__(self, match): url = self.root + "/api/gallery/" + match.group(1) @@ -127,34 +105,18 @@ class NhentaiTagExtractor(NhentaiExtractor): pattern = (r"(?:https?://)?nhentai\.net(" r"/(?:artist|category|character|group|language|parody|tag)" r"/[^/?#]+(?:/popular[^/?#]*)?/?)(?:\?([^#]+))?") - test = ( - ("https://nhentai.net/tag/sole-female/", { - "pattern": NhentaiGalleryExtractor.pattern, - "count": 30, - "range": "1-30", - }), - ("https://nhentai.net/artist/itou-life/"), - ("https://nhentai.net/group/itou-life/"), - ("https://nhentai.net/parody/touhou-project/"), - ("https://nhentai.net/character/patchouli-knowledge/popular"), - ("https://nhentai.net/category/doujinshi/popular-today"), - ("https://nhentai.net/language/english/popular-week"), - ) + example = "https://nhentai.net/tag/TAG/" class NhentaiSearchExtractor(NhentaiExtractor): """Extractor for nhentai search results""" subcategory = "search" pattern = r"(?:https?://)?nhentai\.net(/search/?)\?([^#]+)" - test = ("https://nhentai.net/search/?q=touhou", { - "pattern": NhentaiGalleryExtractor.pattern, - "count": 30, - "range": "1-30", - }) + example = "https://nhentai.net/search/?q=QUERY" class NhentaiFavoriteExtractor(NhentaiExtractor): """Extractor for nhentai favorites""" subcategory = "favorite" pattern = r"(?:https?://)?nhentai\.net(/favorites/?)(?:\?([^#]+))?" - test = ("https://nhentai.net/favorites/",) + example = "https://nhentai.net/favorites/" diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index 66040d8a..9dd9d7ab 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -176,10 +176,7 @@ class NijieUserExtractor(NijieExtractor): subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/members\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members.php?id=44"), - ("https://horne.red/members.php?id=58000"), - ) + example = "https://nijie.info/members.php?id=12345" def initialize(self): pass @@ -198,48 +195,7 @@ class NijieIllustrationExtractor(NijieExtractor): """Extractor for all illustrations of a nijie-user""" subcategory = "illustration" pattern = BASE_PATTERN + r"/members_illust\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members_illust.php?id=44", { - "url": "1553e5144df50a676f5947d02469299b401ad6c0", - "keyword": { - "artist_id": 44, - "artist_name": "ED", - "date": "type:datetime", - "description": str, - "extension": "jpg", - "filename": str, - "image_id": int, - "num": int, - "tags": list, - "title": str, - "url": r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", - "user_id": 44, - "user_name": "ED", - }, - }), - ("https://horne.red/members_illust.php?id=58000", { - "pattern": r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+" - r"/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", - "range": "1-20", - "count": 20, - "keyword": { - "artist_id": 58000, - "artist_name": "のえるわ", - "date": "type:datetime", - "description": str, - "image_id": int, - "num": int, - "tags": list, - "title": str, - "url": str, - "user_id": 58000, - "user_name": "のえるわ", - }, - }), - ("https://nijie.info/members_illust.php?id=43", { - "exception": exception.NotFoundError, - }), - ) + example = "https://nijie.info/members_illust.php?id=12345" def image_ids(self): return self._pagination("members_illust") @@ -249,16 +205,7 @@ class NijieDoujinExtractor(NijieExtractor): """Extractor for doujin entries of a nijie user""" subcategory = "doujin" pattern = BASE_PATTERN + r"/members_dojin\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members_dojin.php?id=6782", { - "count": ">= 18", - "keyword": { - "user_id" : 6782, - "user_name": "ジョニー＠アビオン村", - }, - }), - ("https://horne.red/members_dojin.php?id=58000"), - ) + example = "https://nijie.info/members_dojin.php?id=12345" def image_ids(self): return self._pagination("members_dojin") @@ -270,23 +217,7 @@ class NijieFavoriteExtractor(NijieExtractor): directory_fmt = ("{category}", "bookmarks", "{user_id}") archive_fmt = "f_{user_id}_{image_id}_{num}" pattern = BASE_PATTERN + r"/user_like_illust_view\.php\?id=(\d+)" - test = ( - ("https://nijie.info/user_like_illust_view.php?id=44", { - "count": ">= 16", - "keyword": { - "user_id" : 44, - "user_name": "ED", - }, - }), - ("https://horne.red/user_like_illust_view.php?id=58000", { - "range": "1-5", - "count": 5, - "keyword": { - "user_id" : 58000, - "user_name": "のえるわ", - }, - }), - ) + example = "https://nijie.info/user_like_illust_view.php?id=12345" def image_ids(self): return self._pagination("user_like_illust_view") @@ -304,17 +235,7 @@ class NijieNuitaExtractor(NijieExtractor): directory_fmt = ("{category}", "nuita", "{user_id}") archive_fmt = "n_{user_id}_{image_id}_{num}" pattern = BASE_PATTERN + r"/history_nuita\.php\?id=(\d+)" - test = ( - ("https://nijie.info/history_nuita.php?id=728995", { - "range": "1-10", - "count": 10, - "keyword": { - "user_id" : 728995, - "user_name": "莚", - }, - }), - ("https://horne.red/history_nuita.php?id=58000"), - ) + example = "https://nijie.info/history_nuita.php?id=12345" def image_ids(self): return self._pagination("history_nuita") @@ -334,13 +255,7 @@ class NijieFeedExtractor(NijieExtractor): """Extractor for nijie liked user feed""" subcategory = "feed" pattern = BASE_PATTERN + r"/like_user_view\.php" - test = ( - ("https://nijie.info/like_user_view.php", { - "range": "1-10", - "count": 10, - }), - ("https://horne.red/like_user_view.php"), - ) + example = "https://nijie.info/like_user_view.php" def image_ids(self): return self._pagination("like_user_view") @@ -350,14 +265,11 @@ class NijieFeedExtractor(NijieExtractor): return "" -class NijiefollowedExtractor(NijieExtractor): +class NijieFollowedExtractor(NijieExtractor): """Extractor for followed nijie users""" subcategory = "followed" pattern = BASE_PATTERN + r"/like_my\.php" - test = ( - ("https://nijie.info/like_my.php"), - ("https://horne.red/like_my.php"), - ) + example = "https://nijie.info/like_my.php" def items(self): self.login() @@ -383,32 +295,7 @@ class NijieImageExtractor(NijieExtractor): """Extractor for a nijie work/image""" subcategory = "image" pattern = BASE_PATTERN + r"/view(?:_popup)?\.php\?id=(\d+)" - test = ( - ("https://nijie.info/view.php?id=70720", { - "url": "3d654e890212ba823c9647754767336aebc0a743", - "keyword": "41da5d0e178b04f01fe72460185df52fadc3c91b", - "content": "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", - }), - ("https://nijie.info/view.php?id=70724", { - "count": 0, - }), - ("https://nijie.info/view_popup.php?id=70720"), - ("https://horne.red/view.php?id=8716", { - "count": 4, - "keyword": { - "artist_id": 58000, - "artist_name": "のえるわ", - "date": "dt:2018-02-04 14:47:24", - "description": "ノエル「そんなことしなくても、" - "言ってくれたら咥えるのに・・・♡」", - "image_id": 8716, - "tags": ["男の娘", "フェラ", "オリキャラ", "うちのこ"], - "title": "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", - "user_id": 58000, - "user_name": "のえるわ", - }, - }), - ) + example = "https://nijie.info/view.php?id=12345" def __init__(self, match): NijieExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/nitter.py b/gallery_dl/extractor/nitter.py index fda169d8..9f5cc9da 100644 --- a/gallery_dl/extractor/nitter.py +++ b/gallery_dl/extractor/nitter.py @@ -261,58 +261,7 @@ USER_PATTERN = BASE_PATTERN + r"/(i(?:/user/|d:)(\d+)|[^/?#]+)" class NitterTweetsExtractor(NitterExtractor): subcategory = "tweets" pattern = USER_PATTERN + r"(?:/tweets)?(?:$|\?|#)" - test = ( - ("https://nitter.net/supernaturepics", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - "count": 20, - "keyword": { - "author": { - "name": "supernaturepics", - "nick": "Nature Pictures" - }, - "comments": int, - "content": str, - "count": 1, - "date": "type:datetime", - "likes": int, - "quotes": int, - "retweets": int, - "tweet_id": r"re:\d+", - "user": { - "date": "dt:2015-01-12 10:25:00", - "description": "The very best nature pictures.", - "favourites_count": int, - "followers_count": int, - "friends_count": int, - "id": "2976459548", - "name": "supernaturepics", - "nick": "Nature Pictures", - "profile_banner": "https://nitter.net/pic/https%3A%2F%2Fpb" - "s.twimg.com%2Fprofile_banners%2F2976459" - "548%2F1421058583%2F1500x500", - "profile_image": "https://nitter.net/pic/pbs.twimg.com%2Fp" - "rofile_images%2F554585280938659841%2FFLV" - "AlX18.jpeg", - "statuses_count": 1568, - "verified": False, - }, - }, - }), - ("https://nitter.lacontrevoie.fr/supernaturepics", { - "url": "54f4b55f2099dcc248f3fb7bfacf1349e08d8e2d", - "pattern": r"https://nitter\.lacontrevoie\.fr/pic/orig" - r"/media%2FCGMNYZvW0AIVoom\.jpg", - "range": "1", - }), - ("https://nitter.1d4.us/supernaturepics", { - "range": "1", - "keyword": {"user": {"id": "2976459548"}}, - }), - ("https://nitter.kavin.rocks/id:2976459548"), - ("https://nitter.unixfox.eu/supernaturepics"), - ) + example = "https://nitter.net/USER" def tweets(self): return self._pagination("") @@ -321,17 +270,7 @@ class NitterTweetsExtractor(NitterExtractor): class NitterRepliesExtractor(NitterExtractor): subcategory = "replies" pattern = USER_PATTERN + r"/with_replies" - test = ( - ("https://nitter.net/supernaturepics/with_replies", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/with_replies"), - ("https://nitter.1d4.us/supernaturepics/with_replies"), - ("https://nitter.kavin.rocks/id:2976459548/with_replies"), - ("https://nitter.unixfox.eu/i/user/2976459548/with_replies"), - ) + example = "https://nitter.net/USER/with_replies" def tweets(self): return self._pagination("/with_replies") @@ -340,21 +279,7 @@ class NitterRepliesExtractor(NitterExtractor): class NitterMediaExtractor(NitterExtractor): subcategory = "media" pattern = USER_PATTERN + r"/media" - test = ( - ("https://nitter.net/supernaturepics/media", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.kavin.rocks/id:2976459548/media", { - "pattern": r"https://nitter\.kavin\.rocks/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/media"), - ("https://nitter.1d4.us/supernaturepics/media"), - ("https://nitter.unixfox.eu/i/user/2976459548/media"), - ) + example = "https://nitter.net/USER/media" def tweets(self): return self._pagination("/media") @@ -363,17 +288,7 @@ class NitterMediaExtractor(NitterExtractor): class NitterSearchExtractor(NitterExtractor): subcategory = "search" pattern = USER_PATTERN + r"/search" - test = ( - ("https://nitter.net/supernaturepics/search", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/search"), - ("https://nitter.1d4.us/supernaturepics/search"), - ("https://nitter.kavin.rocks/id:2976459548/search"), - ("https://nitter.unixfox.eu/i/user/2976459548/search"), - ) + example = "https://nitter.net/USER/search" def tweets(self): return self._pagination("/search") @@ -386,95 +301,7 @@ class NitterTweetExtractor(NitterExtractor): filename_fmt = "{tweet_id}_{num}.{extension}" archive_fmt = "{tweet_id}_{num}" pattern = BASE_PATTERN + r"/(i/web|[^/?#]+)/status/(\d+())" - test = ( - ("https://nitter.net/supernaturepics/status/604341487988576256", { - "url": "3f2b64e175bf284aa672c3bb53ed275e470b919a", - "content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", - "keyword": { - "comments": 19, - "content": "Big Wedeene River, Canada", - "count": 1, - "date": "dt:2015-05-29 17:40:00", - "extension": "jpg", - "filename": "CGMNYZvW0AIVoom", - "likes": int, - "num": 1, - "quotes": 10, - "retweets": int, - "tweet_id": "604341487988576256", - "url": "https://nitter.net/pic/orig" - "/media%2FCGMNYZvW0AIVoom.jpg", - "user": { - "name": "supernaturepics", - "nick": "Nature Pictures", - }, - }, - }), - # 4 images - ("https://nitter.lacontrevoie.fr/i/status/894001459754180609", { - "url": "9c51b3a4a1114535eb9b168bba97ad95db0d59ff", - }), - # video - ("https://nitter.lacontrevoie.fr/i/status/1065692031626829824", { - "pattern": r"ytdl:https://nitter\.lacontrevoie\.fr/video" - r"/[0-9A-F]{10,}/https%3A%2F%2Fvideo.twimg.com%2F" - r"ext_tw_video%2F1065691868439007232%2Fpu%2Fpl%2F" - r"nv8hUQC1R0SjhzcZ.m3u8%3Ftag%3D5", - "keyword": { - "extension": "mp4", - "filename": "nv8hUQC1R0SjhzcZ", - }, - }), - # content with emoji, newlines, hashtags (#338) - ("https://nitter.1d4.us/playpokemon/status/1263832915173048321", { - "keyword": {"content": ( - r"re:Gear up for #PokemonSwordShieldEX with special Mystery " - "Gifts! \n\nYou’ll be able to receive four Galarian form " - "Pokémon with Hidden Abilities, plus some very useful items. " - "It’s our \\(Mystery\\) Gift to you, Trainers! \n\n❓🎁➡️ " - )}, - }), - # Nitter tweet (#890) - ("https://nitter.kavin.rocks/ed1conf/status/1163841619336007680", { - "url": "e115bd1c86c660064e392b05269bbcafcd8c8b7a", - "content": "f29501e44d88437fe460f5c927b7543fda0f6e34", - }), - # Reply to deleted tweet (#403, #838) - ("https://nitter.unixfox.eu/i/web/status/1170041925560258560", { - "pattern": r"https://nitter\.unixfox\.eu/pic/orig" - r"/media%2FEDzS7VrU0AAFL4_\.jpg", - }), - # "quoted" option (#854) - ("https://nitter.net/StobiesGalaxy/status/1270755918330896395", { - "options": (("quoted", True),), - "pattern": r"https://nitter\.net/pic/orig/media%2FEa[KG].+\.jpg", - "count": 8, - }), - # quoted tweet (#526, #854) - ("https://nitter.1d4.us/StobiesGalaxy/status/1270755918330896395", { - "pattern": r"https://nitter\.1d4\.us/pic/orig" - r"/enc/bWVkaWEvRWFL\w+LmpwZw==", - "keyword": {"filename": r"re:EaK.{12}"}, - "count": 4, - }), - # deleted quote tweet (#2225) - ("https://nitter.lacontrevoie.fr/i/status/1460044411165888515", { - "count": 0, - }), - # "Misleading" content - ("https://nitter.lacontrevoie.fr/i/status/1486373748911575046", { - "count": 4, - }), - # age-restricted (#2354) - ("https://nitter.unixfox.eu/mightbecurse/status/1492954264909479936", { - "keyword": {"date": "dt:2022-02-13 20:10:00"}, - "count": 1, - }), - # broadcast - ("https://nitter.it/POTUS/status/1639409307878928384", { - "count": 0, - }) - ) + example = "https://nitter.net/USER/status/12345" def tweets(self): url = "{}/i/status/{}".format(self.root, self.user) diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index af2a3678..9bff92ff 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -105,52 +105,7 @@ class NozomiPostExtractor(NozomiExtractor): """Extractor for individual posts on nozomi.la""" subcategory = "post" pattern = r"(?:https?://)?nozomi\.la/post/(\d+)" - test = ( - ("https://nozomi.la/post/3649262.html", { - "url": "e5525e717aec712843be8b88592d6406ae9e60ba", - "pattern": r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb" - r"6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", - "content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", - "keyword": { - "artist" : ["hammer (sunset beach)"], - "character": ["patchouli knowledge"], - "copyright": ["touhou"], - "dataid" : "re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", - "date" : "dt:2016-07-26 02:32:03", - "extension": "webp", - "filename" : str, - "height" : 768, - "is_video" : False, - "postid" : 3649262, - "tags" : list, - "type" : "jpg", - "url" : str, - "width" : 1024, - }, - }), - # multiple images per post - ("https://nozomi.la/post/25588032.html", { - "url": "fb956ccedcf2cf509739d26e2609e910244aa56c", - "keyword": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", - "count": 7, - }), - # empty 'date' (#1163) - ("https://nozomi.la/post/130309.html", { - "keyword": {"date": None}, - }), - # gif - ("https://nozomi.la/post/1647.html", { - "pattern": r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c1" - r"49db459d76b58a074416c260ed93cc31fa9f0a\.gif", - "content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", - }), - # video - ("https://nozomi.la/post/2269847.html", { - "pattern": r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f" - r"2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", - "content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", - }), - ) + example = "https://nozomi.la/post/12345.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -165,11 +120,7 @@ class NozomiIndexExtractor(NozomiExtractor): subcategory = "index" pattern = (r"(?:https?://)?nozomi\.la/" r"(?:(index(?:-Popular)?)-(\d+)\.html)?(?:$|#|\?)") - test = ( - ("https://nozomi.la/"), - ("https://nozomi.la/index-2.html"), - ("https://nozomi.la/index-Popular-33.html"), - ) + example = "https://nozomi.la/index-1.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -183,11 +134,7 @@ class NozomiTagExtractor(NozomiExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{dataid}" pattern = r"(?:https?://)?nozomi\.la/tag/([^/?#]+)-(\d+)\." - test = ("https://nozomi.la/tag/3:1_aspect_ratio-1.html", { - "pattern": r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", - "count": ">= 25", - "range": "1-25", - }) + example = "https://nozomi.la/tag/TAG-1.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -205,9 +152,7 @@ class NozomiSearchExtractor(NozomiExtractor): directory_fmt = ("{category}", "{search_tags:J }") archive_fmt = "t_{search_tags}_{dataid}" pattern = r"(?:https?://)?nozomi\.la/search\.html\?q=([^&#]+)" - test = ("https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", { - "count": ">= 5", - }) + example = "https://nozomi.la/search.html?q=QUERY" def __init__(self, match): NozomiExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/nsfwalbum.py b/gallery_dl/extractor/nsfwalbum.py index 6433fbd2..7229bde5 100644 --- a/gallery_dl/extractor/nsfwalbum.py +++ b/gallery_dl/extractor/nsfwalbum.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,11 +21,7 @@ class NsfwalbumAlbumExtractor(GalleryExtractor): directory_fmt = ("{category}", "{album_id} {title}") archive_fmt = "{id}" pattern = r"(?:https?://)?(?:www\.)?nsfwalbum\.com(/album/(\d+))" - test = ("https://nsfwalbum.com/album/401611", { - "range": "1-5", - "url": "b0481fc7fad5982da397b6359fbed8421b8ba284", - "keyword": "e98f9b0d473c00000831618d0235863b1dd78294", - }) + example = "https://nsfwalbum.com/album/12345" def __init__(self, match): self.album_id = match.group(2) diff --git a/gallery_dl/extractor/nudecollect.py b/gallery_dl/extractor/nudecollect.py index 31599198..bda5d774 100644 --- a/gallery_dl/extractor/nudecollect.py +++ b/gallery_dl/extractor/nudecollect.py @@ -37,26 +37,8 @@ class NudecollectImageExtractor(NudecollectExtractor): pattern = (r"(?:https?://)?(?:www\.)?nudecollect\.com" r"(/content/([^/?#]+)/image-(\d+)-pics-(\d+)" r"-mirror-(\d+)\.html)") - test = ( - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/image-4-pics-108-mirror-43.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00004-5896498214-43" - r"-9689595623/20201220_Teenpornstorage_Patritcy_Vaness" - r"a_Lesbian_Lust/9879560327/nudecollect\.com\.jpg"), - "keyword": { - "slug" : ("20201220_Teenpornstorage_Patritcy" - "_Vanessa_Lesbian_Lust"), - "title" : ("20201220 Teenpornstorage Patritcy" - " Vanessa Lesbian Lust"), - "num" : 4, - "count" : 108, - "mirror": 43, - }, - }), - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/image-10-pics-108-mirror-43.html")), - ) + example = ("https://www.nudecollect.com/content/12345_TITLE" + "/image-1-pics-108-mirror-1.html") def __init__(self, match): NudecollectExtractor.__init__(self, match) @@ -80,45 +62,8 @@ class NudecollectAlbumExtractor(NudecollectExtractor): pattern = (r"(?:https?://)?(?:www\.)?nudecollect\.com" r"/content/([^/?#]+)/(?:index-mirror-(\d+)-(\d+)" r"|page-\d+-pics-(\d+)-mirror-(\d+))\.html") - test = ( - (("https://www.nudecollect.com/content/20170219_TheWhiteBoxxx_" - "Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex" - "_with_alluring_Czech_babes_x125_1080px/index-mirror-67-125.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00\d\d\d-5896498214-67" - r"-9689595623/20170219_TheWhiteBoxxx_Caprice" - r"_Tracy_Loves_Hot_ass_fingering_and_sensual_" - r"lesbian_sex_with_alluring_Czech_babes_x125_1080px" - r"/9879560327/nudecollect\.com\.jpg"), - "count" : 125, - "keyword": { - "slug" : ("20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_" - "ass_fingering_and_sensual_lesbian_sex_with_" - "alluring_Czech_babes_x125_1080px"), - "title" : ("20170219 TheWhiteBoxxx Caprice Tracy Loves Hot ass" - " fingering and sensual lesbian sex with alluring" - " Czech babes x125 1080px"), - "num" : int, - "mirror": 67, - }, - }), - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/page-1-pics-108-mirror-43.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00\d\d\d-5896498214-43" - r"-9689595623/20201220_Teenpornstorage_Patritcy_Vaness" - r"a_Lesbian_Lust/9879560327/nudecollect\.com\.jpg"), - "count" : 108, - "keyword": { - "slug" : ("20201220_Teenpornstorage_Patritcy" - "_Vanessa_Lesbian_Lust"), - "title" : ("20201220 Teenpornstorage Patritcy" - " Vanessa Lesbian Lust"), - "num" : int, - "mirror": 43, - }, - }), - ) + example = ("https://www.nudecollect.com/content/12345_TITLE" + "/index-mirror-01-123.html") def __init__(self, match): self.slug = match.group(1) diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index f109d258..45313c51 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -243,6 +243,7 @@ class OAuthBase(Extractor): class OAuthFlickr(OAuthBase): subcategory = "flickr" pattern = "oauth:flickr$" + example = "oauth:flickr" redirect_uri = REDIRECT_URI_HTTPS def items(self): @@ -261,6 +262,7 @@ class OAuthFlickr(OAuthBase): class OAuthSmugmug(OAuthBase): subcategory = "smugmug" pattern = "oauth:smugmug$" + example = "oauth:smugmug" def items(self): yield Message.Version, 1 @@ -278,6 +280,7 @@ class OAuthSmugmug(OAuthBase): class OAuthTumblr(OAuthBase): subcategory = "tumblr" pattern = "oauth:tumblr$" + example = "oauth:tumblr" def items(self): yield Message.Version, 1 @@ -298,6 +301,7 @@ class OAuthTumblr(OAuthBase): class OAuthDeviantart(OAuthBase): subcategory = "deviantart" pattern = "oauth:deviantart$" + example = "oauth:deviantart" redirect_uri = REDIRECT_URI_HTTPS def items(self): @@ -319,6 +323,7 @@ class OAuthDeviantart(OAuthBase): class OAuthReddit(OAuthBase): subcategory = "reddit" pattern = "oauth:reddit$" + example = "oauth:reddit" def items(self): yield Message.Version, 1 @@ -340,6 +345,7 @@ class OAuthReddit(OAuthBase): class OAuthMastodon(OAuthBase): subcategory = "mastodon" pattern = "oauth:mastodon:(?:https?://)?([^/?#]+)" + example = "oauth:mastodon:mastodon.social" def __init__(self, match): OAuthBase.__init__(self, match) @@ -397,6 +403,7 @@ class OAuthMastodon(OAuthBase): class OAuthPixiv(OAuthBase): subcategory = "pixiv" pattern = "oauth:pixiv$" + example = "oauth:pixiv" def items(self): yield Message.Version, 1 diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py index 6bc7b9a9..0389ead9 100644 --- a/gallery_dl/extractor/paheal.py +++ b/gallery_dl/extractor/paheal.py @@ -74,34 +74,7 @@ class PahealTagExtractor(PahealExtractor): directory_fmt = ("{category}", "{search_tags}") pattern = (r"(?:https?://)?(?:rule34|rule63|cosplay)\.paheal\.net" r"/post/list/([^/?#]+)") - test = ( - ("https://rule34.paheal.net/post/list/Ayane_Suzuki/1", { - "pattern": r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20", - "count": ">= 15" - }), - ("https://rule34.paheal.net/post/list/Ayane_Suzuki/1", { - "range": "1", - "options": (("metadata", True),), - "keyword": { - "date": "dt:2018-01-07 07:04:05", - "duration": 0.0, - "extension": "jpg", - "filename": "2446128 - Ayane_Suzuki Idolmaster " - "idolmaster_dearly_stars Zanzi", - "height": 768, - "id": 2446128, - "md5": "b0ceda9d860df1d15b60293a7eb465c1", - "search_tags": "Ayane_Suzuki", - "size": 205312, - "source": "https://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=19957280", - "tags": "Ayane_Suzuki Idolmaster " - "idolmaster_dearly_stars Zanzi", - "uploader": "XXXname", - "width": 1024, - }, - }), - ) + example = "https://rule34.paheal.net/post/list/TAG/1" per_page = 70 def __init__(self, match): @@ -161,61 +134,7 @@ class PahealPostExtractor(PahealExtractor): subcategory = "post" pattern = (r"(?:https?://)?(?:rule34|rule63|cosplay)\.paheal\.net" r"/post/view/(\d+)") - test = ( - ("https://rule34.paheal.net/post/view/481609", { - "pattern": r"https://tulip\.paheal\.net/_images" - r"/bbdc1c33410c2cdce7556c7990be26b7/481609%20-%20" - r"Azumanga_Daioh%20inanimate%20Osaka%20Vuvuzela\.jpg", - "content": "7b924bcf150b352ac75c9d281d061e174c851a11", - "keyword": { - "date": "dt:2010-06-17 15:40:23", - "extension": "jpg", - "file_url": "re:https://tulip.paheal.net/_images/bbdc1c33410c", - "filename": "481609 - Azumanga_Daioh inanimate Osaka Vuvuzela", - "height": 660, - "id": 481609, - "md5": "bbdc1c33410c2cdce7556c7990be26b7", - "size": 157389, - "source": "", - "tags": "Azumanga_Daioh inanimate Osaka Vuvuzela", - "uploader": "CaptainButtface", - "width": 614, - }, - }), - ("https://rule34.paheal.net/post/view/488534", { - "keyword": { - "date": "dt:2010-06-25 13:51:17", - "height": 800, - "md5": "b39edfe455a0381110c710d6ed2ef57d", - "size": 758989, - "source": "http://www.furaffinity.net/view/4057821/", - "tags": "inanimate thelost-dragon Vuvuzela", - "uploader": "leacheate_soup", - "width": 1200, - }, - }), - # video - ("https://rule34.paheal.net/post/view/3864982", { - "pattern": r"https://[\w]+\.paheal\.net/_images/7629fc0ff77e32637d" - r"de5bf4f992b2cb/3864982%20-%20animated%20Metal_Gear%20" - r"Metal_Gear_Solid_V%20Quiet%20Vg_erotica%20webm\.webm", - "keyword": { - "date": "dt:2020-09-06 01:59:03", - "duration": 30.0, - "extension": "webm", - "height": 2500, - "id": 3864982, - "md5": "7629fc0ff77e32637dde5bf4f992b2cb", - "size": 18454938, - "source": "https://twitter.com/VG_Worklog" - "/status/1302407696294055936", - "tags": "animated Metal_Gear Metal_Gear_Solid_V " - "Quiet Vg_erotica webm", - "uploader": "justausername", - "width": 1768, - }, - }), - ) + example = "https://rule34.paheal.net/post/view/12345" def __init__(self, match): PahealExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 99d9457a..0c5dfbe2 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -260,34 +260,7 @@ class PatreonCreatorExtractor(PatreonExtractor): pattern = (r"(?:https?://)?(?:www\.)?patreon\.com" r"/(?!(?:home|join|posts|login|signup)(?:$|[/?#]))" r"([^/?#]+)(?:/posts)?/?(?:\?([^#]+))?") - test = ( - ("https://www.patreon.com/koveliana", { - "range": "1-25", - "count": ">= 25", - "keyword": { - "attachments" : list, - "comment_count": int, - "content" : str, - "creator" : dict, - "date" : "type:datetime", - "id" : int, - "images" : list, - "like_count" : int, - "post_type" : str, - "published_at" : str, - "title" : str, - }, - }), - ("https://www.patreon.com/koveliana/posts?filters[month]=2020-3", { - "count": 1, - "keyword": {"date": "dt:2020-03-30 21:21:44"}, - }), - ("https://www.patreon.com/kovelianot", { - "exception": exception.NotFoundError, - }), - ("https://www.patreon.com/user?u=2931440"), - ("https://www.patreon.com/user/posts/?u=2931440"), - ) + example = "https://www.patreon.com/USER" def __init__(self, match): PatreonExtractor.__init__(self, match) @@ -328,7 +301,7 @@ class PatreonUserExtractor(PatreonExtractor): """Extractor for media from creators supported by you""" subcategory = "user" pattern = r"(?:https?://)?(?:www\.)?patreon\.com/home$" - test = ("https://www.patreon.com/home",) + example = "https://www.patreon.com/home" def posts(self): url = self._build_url("stream", ( @@ -343,24 +316,7 @@ class PatreonPostExtractor(PatreonExtractor): """Extractor for media from a single post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?patreon\.com/posts/([^/?#]+)" - test = ( - # postfile + attachments - ("https://www.patreon.com/posts/precious-metal-23563293", { - "count": 4, - }), - # postfile + content - ("https://www.patreon.com/posts/56127163", { - "count": 3, - "keyword": {"filename": r"re:^(?!1).+$"}, - }), - # tags (#1539) - ("https://www.patreon.com/posts/free-post-12497641", { - "keyword": {"tags": ["AWMedia"]}, - }), - ("https://www.patreon.com/posts/not-found-123", { - "exception": exception.NotFoundError, - }), - ) + example = "https://www.patreon.com/posts/TITLE-12345" def __init__(self, match): PatreonExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index 8fa5de24..3a0f5b02 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -57,68 +57,7 @@ class PhilomenaPostExtractor(PhilomenaExtractor): """Extractor for single posts on a Philomena booru""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:images/)?(\d+)" - test = ( - ("https://derpibooru.org/images/1", { - "content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", - "count": 1, - "keyword": { - "animated": False, - "aspect_ratio": 1.0, - "comment_count": int, - "created_at": "2012-01-02T03:12:33Z", - "date": "dt:2012-01-02 03:12:33", - "deletion_reason": None, - "description": "", - "downvotes": int, - "duplicate_of": None, - "duration": 0.04, - "extension": "png", - "faves": int, - "first_seen_at": "2012-01-02T03:12:33Z", - "format": "png", - "height": 900, - "hidden_from_users": False, - "id": 1, - "mime_type": "image/png", - "name": "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+ga" - "lore_artist-colon-speccysy_get_sunshine", - "orig_sha512_hash": None, - "processed": True, - "representations": dict, - "score": int, - "sha512_hash": "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391" - "aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07e" - "f7800663b994f3f343c572da5ecc22a9660f", - "size": 860914, - "source_url": "https://www.deviantart.com/speccysy/art" - "/Afternoon-Flight-215193985", - "spoilered": False, - "tag_count": int, - "tag_ids": list, - "tags": list, - "thumbnails_generated": True, - "updated_at": r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", - "uploader": "Clover the Clever", - "uploader_id": 211188, - "upvotes": int, - "view_url": str, - "width": 900, - "wilson_score": float, - }, - }), - ("https://derpibooru.org/1"), - ("https://www.derpibooru.org/1"), - ("https://www.derpibooru.org/images/1"), - - ("https://ponybooru.org/images/1", { - "content": "bca26f58fafd791fe07adcd2a28efd7751824605", - }), - ("https://www.ponybooru.org/images/1"), - - ("https://furbooru.org/images/1", { - "content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", - }), - ) + example = "https://derpibooru.org/images/12345" def __init__(self, match): PhilomenaExtractor.__init__(self, match) @@ -133,28 +72,7 @@ class PhilomenaSearchExtractor(PhilomenaExtractor): subcategory = "search" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?:search/?\?([^#]+)|tags/([^/?#]+))" - test = ( - ("https://derpibooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://derpibooru.org/tags/cute", { - "range": "40-60", - "count": 21, - }), - (("https://derpibooru.org/tags/" - "artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak"), { - "count": ">= 2", - }), - ("https://ponybooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://furbooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ) + example = "https://derpibooru.org/search?q=QUERY" def __init__(self, match): PhilomenaExtractor.__init__(self, match) @@ -188,28 +106,7 @@ class PhilomenaGalleryExtractor(PhilomenaExtractor): directory_fmt = ("{category}", "galleries", "{gallery[id]} {gallery[title]}") pattern = BASE_PATTERN + r"/galleries/(\d+)" - test = ( - ("https://derpibooru.org/galleries/1", { - "pattern": r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", - "keyword": { - "gallery": { - "description": "Indexes start at 1 :P", - "id": 1, - "spoiler_warning": "", - "thumbnail_id": 1, - "title": "The Very First Gallery", - "user": "DeliciousBlackInk", - "user_id": 365446, - }, - }, - }), - ("https://ponybooru.org/galleries/27", { - "count": ">= 24", - }), - ("https://furbooru.org/galleries/27", { - "count": ">= 13", - }), - ) + example = "https://derpibooru.org/galleries/12345" def __init__(self, match): PhilomenaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/photobucket.py b/gallery_dl/extractor/photobucket.py index 22aff884..a01c9fe6 100644 --- a/gallery_dl/extractor/photobucket.py +++ b/gallery_dl/extractor/photobucket.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://photobucket.com/""" +"""Extractors for https://photobucket.com/""" from .common import Extractor, Message from .. import text, exception @@ -23,29 +23,7 @@ class PhotobucketAlbumExtractor(Extractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?((?:[\w-]+\.)?photobucket\.com)" r"/user/[^/?&#]+/library(?:/[^?&#]*)?") - test = ( - ("https://s369.photobucket.com/user/CrpyLrkr/library", { - "pattern": r"https?://[oi]+\d+.photobucket.com/albums/oo139/", - "count": ">= 50" - }), - # subalbums of main "directory" - ("https://s271.photobucket.com/user/lakerfanryan/library/", { - "options": (("image-filter", "False"),), - "pattern": pattern, - "count": 1, - }), - # subalbums of subalbum without images - ("https://s271.photobucket.com/user/lakerfanryan/library/Basketball", { - "pattern": pattern, - "count": ">= 9", - }), - # private (missing JSON data) - ("https://s1277.photobucket.com/user/sinisterkat44/library/", { - "count": 0, - }), - ("https://s1110.photobucket.com/user/chndrmhn100/library/" - "Chandu%20is%20the%20King?sort=3&page=1"), - ) + example = "https://s123.photobucket.com/user/USER/library" def __init__(self, match): self.root = "https://" + match.group(1) @@ -113,18 +91,7 @@ class PhotobucketImageExtractor(Extractor): pattern = (r"(?:https?://)?(?:[\w-]+\.)?photobucket\.com" r"(?:/gallery/user/([^/?&#]+)/media/([^/?&#]+)" r"|/user/([^/?&#]+)/media/[^?&#]+\.html)") - test = ( - (("https://s271.photobucket.com/user/lakerfanryan" - "/media/Untitled-3-1.jpg.html"), { - "url": "3b647deeaffc184cc48c89945f67574559c9051f", - "keyword": "69732741b2b351db7ecaa77ace2fdb39f08ca5a3", - }), - (("https://s271.photobucket.com/user/lakerfanryan" - "/media/IsotopeswBros.jpg.html?sort=3&o=2"), { - "url": "12c1890c09c9cdb8a88fba7eec13f324796a8d7b", - "keyword": "61200a223df6c06f45ac3d30c88b3f5b048ce9a8", - }), - ) + example = "https://s123.photobucket.com/user/USER/media/NAME.EXT.html" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/photovogue.py b/gallery_dl/extractor/photovogue.py index 3c68fd51..2a2df5a6 100644 --- a/gallery_dl/extractor/photovogue.py +++ b/gallery_dl/extractor/photovogue.py @@ -19,38 +19,7 @@ class PhotovogueUserExtractor(Extractor): filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/photographers/(\d+)" - test = ( - ("https://www.vogue.com/photovogue/photographers/221252"), - ("https://vogue.com/photovogue/photographers/221252", { - "pattern": r"https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - "keyword": { - "date": "type:datetime", - "favorite_count": int, - "favorited": list, - "id": int, - "image_id": str, - "is_favorite": False, - "orientation": "re:portrait|landscape", - "photographer": { - "biography": "Born in 1995. Live in Bologna.", - "city": "Bologna", - "country_id": 106, - "favoritedCount": int, - "id": 221252, - "isGold": bool, - "isPro": bool, - "latitude": str, - "longitude": str, - "name": "Arianna Mattarozzi", - "user_id": "38cb0601-4a85-453c-b7dc-7650a037f2ab", - "websites": list, - }, - "photographer_id": 221252, - "tags": list, - "title": str, - }, - }), - ) + example = "https://www.vogue.com/photovogue/photographers/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/picarto.py b/gallery_dl/extractor/picarto.py index 77a07b48..cc7eee5f 100644 --- a/gallery_dl/extractor/picarto.py +++ b/gallery_dl/extractor/picarto.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,12 +21,7 @@ class PicartoGalleryExtractor(Extractor): filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?picarto\.tv/([^/?#]+)/gallery" - test = ("https://picarto.tv/fnook/gallery/default/", { - "pattern": r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork" - r"/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", - "count": ">= 7", - "keyword": {"date": "type:datetime"}, - }) + example = "https://picarto.tv/USER/gallery/TITLE/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/piczel.py b/gallery_dl/extractor/piczel.py index 56c29782..422325ff 100644 --- a/gallery_dl/extractor/piczel.py +++ b/gallery_dl/extractor/piczel.py @@ -68,10 +68,7 @@ class PiczelUserExtractor(PiczelExtractor): """Extractor for all images from a user's gallery""" subcategory = "user" pattern = r"(?:https?://)?(?:www\.)?piczel\.tv/gallery/([^/?#]+)/?$" - test = ("https://piczel.tv/gallery/Bikupan", { - "range": "1-100", - "count": ">= 100", - }) + example = "https://piczel.tv/gallery/USER" def __init__(self, match): PiczelExtractor.__init__(self, match) @@ -89,9 +86,7 @@ class PiczelFolderExtractor(PiczelExtractor): archive_fmt = "f{folder[id]}_{id}_{num}" pattern = (r"(?:https?://)?(?:www\.)?piczel\.tv" r"/gallery/(?!image)([^/?#]+)/(\d+)") - test = ("https://piczel.tv/gallery/Lulena/1114", { - "count": ">= 4", - }) + example = "https://piczel.tv/gallery/USER/12345" def __init__(self, match): PiczelExtractor.__init__(self, match) @@ -106,30 +101,7 @@ class PiczelImageExtractor(PiczelExtractor): """Extractor for individual images""" subcategory = "image" pattern = r"(?:https?://)?(?:www\.)?piczel\.tv/gallery/image/(\d+)" - test = ("https://piczel.tv/gallery/image/7807", { - "pattern": r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image" - r"/32920/image/7807/1532236438-Lulena\.png", - "content": "df9a053a24234474a19bce2b7e27e0dec23bff87", - "keyword": { - "created_at": "2018-07-22T05:13:58.000Z", - "date": "dt:2018-07-22 05:13:58", - "description": None, - "extension": "png", - "favorites_count": int, - "folder_id": 1113, - "id": 7807, - "is_flash": False, - "is_video": False, - "multi": False, - "nsfw": False, - "num": 0, - "password_protected": False, - "tags": ["fanart", "commission", "altair", "recreators"], - "title": "Altair", - "user": dict, - "views": int, - }, - }) + example = "https://piczel.tv/gallery/image/12345" def __init__(self, match): PiczelExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pillowfort.py b/gallery_dl/extractor/pillowfort.py index af7d57f1..8cfefa97 100644 --- a/gallery_dl/extractor/pillowfort.py +++ b/gallery_dl/extractor/pillowfort.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -122,69 +122,7 @@ class PillowfortPostExtractor(PillowfortExtractor): """Extractor for a single pillowfort post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.pillowfort.social/posts/27510", { - "pattern": r"https://img\d+\.pillowfort\.social" - r"/posts/\w+_out\d+\.png", - "count": 4, - "keyword": { - "avatar_url": str, - "col": 0, - "commentable": True, - "comments_count": int, - "community_id": None, - "content": str, - "created_at": str, - "date": "type:datetime", - "deleted": None, - "deleted_at": None, - "deleted_by_mod": None, - "deleted_for_flag_id": None, - "embed_code": None, - "id": int, - "last_activity": str, - "last_activity_elapsed": str, - "last_edited_at": str, - "likes_count": int, - "media_type": "picture", - "nsfw": False, - "num": int, - "original_post_id": None, - "original_post_user_id": None, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id": 27510, - "post_type": "picture", - "privacy": "public", - "reblog_copy_info": list, - "rebloggable": True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count": int, - "row": int, - "small_image_url": None, - "tags": list, - "time_elapsed": str, - "timestamp": str, - "title": "What is Pillowfort.social?", - "updated_at": str, - "url": r"re:https://img3.pillowfort.social/posts/.*\.png", - "user_id": 5, - "username": "Staff" - }, - }), - ("https://www.pillowfort.social/posts/1557500", { - "options": (("external", True), ("inline", False)), - "pattern": r"https://twitter\.com/Aliciawitdaart/status" - r"/1282862493841457152", - }), - ("https://www.pillowfort.social/posts/1672518", { - "options": (("inline", True),), - "count": 3, - }), - ) + example = "https://www.pillowfort.social/posts/12345" def posts(self): url = "{}/posts/{}/json/".format(self.root, self.item) @@ -195,11 +133,7 @@ class PillowfortUserExtractor(PillowfortExtractor): """Extractor for all posts of a pillowfort user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!posts/)([^/?#]+)" - test = ("https://www.pillowfort.social/Pome", { - "pattern": r"https://img\d+\.pillowfort\.social/posts/", - "range": "1-15", - "count": 15, - }) + example = "https://www.pillowfort.social/USER" def posts(self): url = "{}/{}/json/".format(self.root, self.item) diff --git a/gallery_dl/extractor/pinterest.py b/gallery_dl/extractor/pinterest.py index be30705b..8afa397e 100644 --- a/gallery_dl/extractor/pinterest.py +++ b/gallery_dl/extractor/pinterest.py @@ -111,21 +111,7 @@ class PinterestPinExtractor(PinterestExtractor): """Extractor for images from a single pin from pinterest.com""" subcategory = "pin" pattern = BASE_PATTERN + r"/pin/([^/?#]+)(?!.*#related$)" - test = ( - ("https://www.pinterest.com/pin/858146903966145189/", { - "url": "afb3c26719e3a530bb0e871c480882a801a4e8a5", - "content": ("4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", - "d3e24bc9f7af585e8c23b9136956bd45a4d9b947"), - }), - # video pin (#1189) - ("https://www.pinterest.com/pin/422564377542934214/", { - "pattern": r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff" - r"/d722ff00ab2352981b89974b37909de8.m3u8", - }), - ("https://www.pinterest.com/pin/858146903966145188/", { - "exception": exception.NotFoundError, - }), - ) + example = "https://www.pinterest.com/pin/12345/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -147,26 +133,7 @@ class PinterestBoardExtractor(PinterestExtractor): archive_fmt = "{board[id]}_{id}" pattern = (BASE_PATTERN + r"/(?!pin/)([^/?#]+)" "/(?!_saved|_created|pins/)([^/?#]+)/?$") - test = ( - ("https://www.pinterest.com/g1952849/test-/", { - "pattern": r"https://i\.pinimg\.com/originals/", - "count": 2, - }), - # board with sections (#835) - ("https://www.pinterest.com/g1952849/stuff/", { - "options": (("sections", True),), - "count": 4, - }), - # secret board (#1055) - ("https://www.pinterest.de/g1952849/secret/", { - "count": 2, - }), - ("https://www.pinterest.com/g1952848/test/", { - "exception": exception.GalleryDLException, - }), - # .co.uk TLD (#914) - ("https://www.pinterest.co.uk/hextra7519/based-animals/"), - ) + example = "https://www.pinterest.com/USER/BOARD/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -197,13 +164,7 @@ class PinterestUserExtractor(PinterestExtractor): """Extractor for a user's boards""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)(?:/_saved)?/?$" - test = ( - ("https://www.pinterest.com/g1952849/", { - "pattern": PinterestBoardExtractor.pattern, - "count": ">= 2", - }), - ("https://www.pinterest.com/g1952849/_saved/"), - ) + example = "https://www.pinterest.com/USER/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -222,11 +183,7 @@ class PinterestAllpinsExtractor(PinterestExtractor): subcategory = "allpins" directory_fmt = ("{category}", "{user}") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/pins/?$" - test = ("https://www.pinterest.com/g1952849/pins/", { - "pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", - "count": 7, - }) + example = "https://www.pinterest.com/USER/pins/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -244,12 +201,7 @@ class PinterestCreatedExtractor(PinterestExtractor): subcategory = "created" directory_fmt = ("{category}", "{user}") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/_created/?$" - test = ("https://www.pinterest.de/digitalmomblog/_created/", { - "pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png)", - "count": 10, - "range": "1-10", - }) + example = "https://www.pinterest.com/USER/_created/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -269,9 +221,7 @@ class PinterestSectionExtractor(PinterestExtractor): "{board[name]}", "{section[title]}") archive_fmt = "{board[id]}_{id}" pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/([^/?#]+)/([^/?#]+)" - test = ("https://www.pinterest.com/g1952849/stuff/section", { - "count": 2, - }) + example = "https://www.pinterest.com/USER/BOARD/SEcTION" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -299,10 +249,7 @@ class PinterestSearchExtractor(PinterestExtractor): subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search/pins/?\?q=([^&#]+)" - test = ("https://www.pinterest.com/search/pins/?q=nature", { - "range": "1-50", - "count": ">= 50", - }) + example = "https://www.pinterest.com/search/pins/?q=QUERY" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -320,11 +267,7 @@ class PinterestRelatedPinExtractor(PinterestPinExtractor): subcategory = "related-pin" directory_fmt = ("{category}", "related {original_pin[id]}") pattern = BASE_PATTERN + r"/pin/([^/?#]+).*#related$" - test = ("https://www.pinterest.com/pin/858146903966145189/#related", { - "range": "31-70", - "count": 40, - "archive": False, - }) + example = "https://www.pinterest.com/pin/12345/#related" def metadata(self): return {"original_pin": self.api.pin(self.pin_id)} @@ -339,11 +282,7 @@ class PinterestRelatedBoardExtractor(PinterestBoardExtractor): directory_fmt = ("{category}", "{board[owner][username]}", "{board[name]}", "related") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/([^/?#]+)/?#related$" - test = ("https://www.pinterest.com/g1952849/test-/#related", { - "range": "31-70", - "count": 40, - "archive": False, - }) + example = "https://www.pinterest.com/USER/BOARD/#related" def pins(self): return self.api.board_content_recommendation(self.board["id"]) @@ -353,15 +292,7 @@ class PinterestPinitExtractor(PinterestExtractor): """Extractor for images from a pin.it URL""" subcategory = "pinit" pattern = r"(?:https?://)?pin\.it/([^/?#]+)" - - test = ( - ("https://pin.it/Hvt8hgT", { - "url": "8daad8558382c68f0868bdbd17d05205184632fa", - }), - ("https://pin.it/Hvt8hgS", { - "exception": exception.NotFoundError, - }), - ) + example = "https://pin.it/abcde" def __init__(self, match): PinterestExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index fc5f39c4..18a3ceb6 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -161,13 +161,7 @@ class PixivUserExtractor(PixivExtractor): pattern = (BASE_PATTERN + r"/(?:" r"(?:en/)?u(?:sers)?/|member\.php\?id=|(?:mypage\.php)?#id=" r")(\d+)(?:$|[?#])") - test = ( - ("https://www.pixiv.net/en/users/173530"), - ("https://www.pixiv.net/u/173530"), - ("https://www.pixiv.net/member.php?id=173530"), - ("https://www.pixiv.net/mypage.php#id=173530"), - ("https://www.pixiv.net/#id=173530"), - ) + example = "https://www.pixiv.net/en/users/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -195,29 +189,7 @@ class PixivArtworksExtractor(PixivExtractor): r"(?:en/)?users/(\d+)/(?:artworks|illustrations|manga)" r"(?:/([^/?#]+))?/?(?:$|[?#])" r"|member_illust\.php\?id=(\d+)(?:&([^#]+))?)") - test = ( - ("https://www.pixiv.net/en/users/173530/artworks", { - "url": "852c31ad83b6840bacbce824d85f2a997889efb7", - }), - # illusts with specific tag - (("https://www.pixiv.net/en/users/173530/artworks" - "/%E6%89%8B%E3%81%B6%E3%82%8D"), { - "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", - }), - (("https://www.pixiv.net/member_illust.php?id=173530" - "&tag=%E6%89%8B%E3%81%B6%E3%82%8D"), { - "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", - }), - # deleted account - ("http://www.pixiv.net/member_illust.php?id=173531", { - "options": (("metadata", True),), - "exception": exception.NotFoundError, - }), - ("https://www.pixiv.net/en/users/173530/manga"), - ("https://www.pixiv.net/en/users/173530/illustrations"), - ("https://www.pixiv.net/member_illust.php?id=173530"), - ("https://touch.pixiv.net/member_illust.php?id=173530"), - ) + example = "https://www.pixiv.net/en/users/12345/artworks" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -253,9 +225,7 @@ class PixivAvatarExtractor(PixivExtractor): filename_fmt = "avatar{date:?_//%Y-%m-%d}.{extension}" archive_fmt = "avatar_{user[id]}_{date}" pattern = USER_PATTERN + r"/avatar" - test = ("https://www.pixiv.net/en/users/173530/avatar", { - "content": "4e57544480cc2036ea9608103e8f024fa737fe66", - }) + example = "https://www.pixiv.net/en/users/12345/avatar" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -273,10 +243,7 @@ class PixivBackgroundExtractor(PixivExtractor): filename_fmt = "background{date:?_//%Y-%m-%d}.{extension}" archive_fmt = "background_{user[id]}_{date}" pattern = USER_PATTERN + "/background" - test = ("https://www.pixiv.net/en/users/194921/background", { - "pattern": r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02" - r"/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", - }) + example = "https://www.pixiv.net/en/users/12345/background" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -303,14 +270,7 @@ class PixivMeExtractor(PixivExtractor): """Extractor for pixiv.me URLs""" subcategory = "me" pattern = r"(?:https?://)?pixiv\.me/([^/?#]+)" - test = ( - ("https://pixiv.me/del_shannon", { - "url": "29c295ce75150177e6b0a09089a949804c708fbf", - }), - ("https://pixiv.me/del_shanno", { - "exception": exception.NotFoundError, - }), - ) + example = "https://pixiv.me/USER" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -333,45 +293,7 @@ class PixivWorkExtractor(PixivExtractor): r"|(?:i(?:\d+\.pixiv|\.pximg)\.net" r"/(?:(?:.*/)?img-[^/]+/img/\d{4}(?:/\d\d){5}|img\d+/img/[^/]+)" r"|img\d*\.pixiv\.net/img/[^/]+|(?:www\.)?pixiv\.net/i)/(\d+))") - test = ( - ("https://www.pixiv.net/artworks/966412", { - "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", - "keyword": { - "date" : "dt:2008-06-12 15:29:13", - "date_url": "dt:2008-06-12 15:29:13", - }, - }), - (("http://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=966411"), { - "exception": exception.NotFoundError, - }), - # ugoira - (("https://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=66806629"), { - "url": "7267695a985c4db8759bebcf8d21dbdd2d2317ef", - "keyword": { - "frames" : list, - "date" : "dt:2018-01-14 15:06:08", - "date_url": "dt:2018-01-15 04:24:48", - }, - }), - # related works (#1237) - ("https://www.pixiv.net/artworks/966412", { - "options": (("related", True),), - "range": "1-10", - "count": ">= 10", - }), - ("https://www.pixiv.net/en/artworks/966412"), - ("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641"), - ("http://i1.pixiv.net/c/600x600/img-master" - "/img/2008/06/13/00/29/13/966412_p0_master1200.jpg"), - ("https://i.pximg.net/img-original" - "/img/2017/04/25/07/33/29/62568267_p0.png"), - ("https://www.pixiv.net/i/966412"), - ("http://img.pixiv.net/img/soundcross/42626136.jpg"), - ("http://i2.pixiv.net/img76/img/snailrin/42672235.jpg"), - ) + example = "https://www.pixiv.net/artworks/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -394,47 +316,7 @@ class PixivFavoriteExtractor(PixivExtractor): pattern = (BASE_PATTERN + r"/(?:(?:en/)?" r"users/(\d+)/(bookmarks/artworks|following)(?:/([^/?#]+))?" r"|bookmark\.php)(?:\?([^#]*))?") - test = ( - ("https://www.pixiv.net/en/users/173530/bookmarks/artworks", { - "url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", - }), - ("https://www.pixiv.net/bookmark.php?id=173530", { - "url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", - }), - # bookmarks with specific tag - (("https://www.pixiv.net/en/users/3137110" - "/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93"), { - "url": "379b28275f786d946e01f721e54afe346c148a8c", - }), - # bookmarks with specific tag (legacy url) - (("https://www.pixiv.net/bookmark.php?id=3137110" - "&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1"), { - "url": "379b28275f786d946e01f721e54afe346c148a8c", - }), - # own bookmarks - ("https://www.pixiv.net/bookmark.php", { - "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "keyword": {"tags_bookmark": ["47", "hitman"]}, - "options": (("metadata-bookmark", True),), - }), - # own bookmarks with tag (#596) - ("https://www.pixiv.net/bookmark.php?tag=foobar", { - "count": 0, - }), - # followed users (#515) - ("https://www.pixiv.net/en/users/173530/following", { - "pattern": PixivUserExtractor.pattern, - "count": ">= 12", - }), - # followed users (legacy url) (#515) - ("https://www.pixiv.net/bookmark.php?id=173530&type=user", { - "pattern": PixivUserExtractor.pattern, - "count": ">= 12", - }), - # touch URLs - ("https://touch.pixiv.net/bookmark.php?id=173530"), - ("https://touch.pixiv.net/bookmark.php"), - ) + example = "https://www.pixiv.net/en/users/12345/bookmarks/artworks" def __init__(self, match): uid, kind, self.tag, query = match.groups() @@ -495,14 +377,7 @@ class PixivRankingExtractor(PixivExtractor): directory_fmt = ("{category}", "rankings", "{ranking[mode]}", "{ranking[date]}") pattern = BASE_PATTERN + r"/ranking\.php(?:\?([^#]*))?" - test = ( - ("https://www.pixiv.net/ranking.php?mode=daily&date=20170818"), - ("https://www.pixiv.net/ranking.php"), - ("https://touch.pixiv.net/ranking.php"), - ("https://www.pixiv.net/ranking.php?mode=unknown", { - "exception": exception.StopExtraction, - }), - ) + example = "https://www.pixiv.net/ranking.php" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -561,24 +436,7 @@ class PixivSearchExtractor(PixivExtractor): directory_fmt = ("{category}", "search", "{search[word]}") pattern = (BASE_PATTERN + r"/(?:(?:en/)?tags/([^/?#]+)(?:/[^/?#]+)?/?" r"|search\.php)(?:\?([^#]+))?") - test = ( - ("https://www.pixiv.net/en/tags/Original", { - "range": "1-10", - "count": 10, - }), - ("https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", { - "exception": exception.StopExtraction, - }), - ("https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", { - "exception": exception.StopExtraction, - }), - ("https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", { - "exception": exception.StopExtraction, - }), - ("https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag"), - ("https://www.pixiv.net/search.php?s_mode=s_tag&word=Original"), - ("https://touch.pixiv.net/search.php?word=Original"), - ) + example = "https://www.pixiv.net/en/tags/TAG" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -643,10 +501,7 @@ class PixivFollowExtractor(PixivExtractor): archive_fmt = "F_{user_follow[id]}_{id}{num}.{extension}" directory_fmt = ("{category}", "following") pattern = BASE_PATTERN + r"/bookmark_new_illust\.php" - test = ( - ("https://www.pixiv.net/bookmark_new_illust.php"), - ("https://touch.pixiv.net/bookmark_new_illust.php"), - ) + example = "https://www.pixiv.net/bookmark_new_illust.php" def works(self): return self.api.illust_follow() @@ -663,17 +518,7 @@ class PixivPixivisionExtractor(PixivExtractor): "{pixivision_id} {pixivision_title}") archive_fmt = "V{pixivision_id}_{id}{suffix}.{extension}" pattern = r"(?:https?://)?(?:www\.)?pixivision\.net/(?:en/)?a/(\d+)" - test = ( - ("https://www.pixivision.net/en/a/2791"), - ("https://pixivision.net/a/2791", { - "count": 7, - "keyword": { - "pixivision_id": "2791", - "pixivision_title": "What's your favorite music? Editor’s " - "picks featuring: “CD Covers”!", - }, - }), - ) + example = "https://www.pixivision.net/en/a/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -705,22 +550,7 @@ class PixivSeriesExtractor(PixivExtractor): "{series[id]} {series[title]}") filename_fmt = "{num_series:>03}_{id}_p{num}.{extension}" pattern = BASE_PATTERN + r"/user/(\d+)/series/(\d+)" - test = ("https://www.pixiv.net/user/10509347/series/21859", { - "range": "1-10", - "count": 10, - "keyword": { - "num_series": int, - "series": { - "canonical": "https://www.pixiv.net/user/10509347" - "/series/21859", - "description": str, - "ogp": dict, - "title": "先輩がうざい後輩の話", - "total": int, - "twitter": dict, - }, - }, - }) + example = "https://www.pixiv.net/user/12345/series/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -762,56 +592,7 @@ class PixivNovelExtractor(PixivExtractor): subcategory = "novel" request_interval = 1.0 pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)" - test = ( - ("https://www.pixiv.net/novel/show.php?id=19612040", { - "count": 1, - "content": "8c818474153cbd2f221ee08766e1d634c821d8b4", - "keyword": { - "caption": r"re:「無能な名無し」と呼ばれ虐げられて育った鈴\(すず\)は、", - "comment_access_control": 0, - "create_date": "2023-04-02T15:18:58+09:00", - "date": "dt:2023-04-02 06:18:58", - "id": 19612040, - "is_bookmarked": False, - "is_muted": False, - "is_mypixiv_only": False, - "is_original": True, - "is_x_restricted": False, - "novel_ai_type": 1, - "page_count": 1, - "rating": "General", - "restrict": 0, - "series": { - "id": 10278364, - "title": "龍の贄嫁〜無能な名無しと虐げられていましたが、" - "どうやら異母妹に霊力を搾取されていたようです〜", - }, - "tags": ["和風ファンタジー", "溺愛", "神様", "ヤンデレ", "執着", - "異能", "ざまぁ", "学園", "神嫁"], - "text_length": 5974, - "title": "異母妹から「無能な名無し」と虐げられていた私、" - "どうやら異母妹に霊力を搾取されていたようです（１）", - "user": { - "account": "yukinaga_chifuyu", - "id": 77055466, - }, - "visible": True, - "x_restrict": 0, - }, - }), - # embeds - ("https://www.pixiv.net/novel/show.php?id=16422450", { - "options": (("embeds", True),), - "count": 3, - }), - # full series - ("https://www.pixiv.net/novel/show.php?id=19612040", { - "options": (("full-series", True),), - "count": 4, - }), - # short URL - ("https://www.pixiv.net/n/19612040"), - ) + example = "https://www.pixiv.net/novel/show.php?id=12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -922,11 +703,7 @@ class PixivNovelUserExtractor(PixivNovelExtractor): """Extractor for pixiv users' novels""" subcategory = "novel-user" pattern = USER_PATTERN + r"/novels" - test = ("https://www.pixiv.net/en/users/77055466/novels", { - "pattern": "^text:", - "range": "1-5", - "count": 5, - }) + example = "https://www.pixiv.net/en/users/12345/novels" def novels(self): return self.api.user_novels(self.novel_id) @@ -936,10 +713,7 @@ class PixivNovelSeriesExtractor(PixivNovelExtractor): """Extractor for pixiv novel series""" subcategory = "novel-series" pattern = BASE_PATTERN + r"/novel/series/(\d+)" - test = ("https://www.pixiv.net/novel/series/10278364", { - "count": 4, - "content": "b06abed001b3f6ccfb1579699e9a238b46d38ea2", - }) + example = "https://www.pixiv.net/novel/series/12345" def novels(self): return self.api.novel_series(self.novel_id) @@ -950,13 +724,7 @@ class PixivNovelBookmarkExtractor(PixivNovelExtractor): subcategory = "novel-bookmark" pattern = (USER_PATTERN + r"/bookmarks/novels" r"(?:/([^/?#]+))?(?:/?\?([^#]+))?") - test = ( - ("https://www.pixiv.net/en/users/77055466/bookmarks/novels", { - "count": 1, - "content": "7194e8faa876b2b536f185ee271a2b6e46c69089", - }), - ("https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide"), - ) + example = "https://www.pixiv.net/en/users/12345/bookmarks/novels" def __init__(self, match): PixivNovelExtractor.__init__(self, match) @@ -986,11 +754,7 @@ class PixivSketchExtractor(Extractor): root = "https://sketch.pixiv.net" cookies_domain = ".pixiv.net" pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)" - test = ("https://sketch.pixiv.net/@nicoby", { - "pattern": r"https://img\-sketch\.pixiv\.net/uploads/medium" - r"/file/\d+/\d+\.(jpg|png)", - "count": ">= 35", - }) + example = "https://sketch.pixiv.net/@USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/pixnet.py b/gallery_dl/extractor/pixnet.py index 7a5009cf..eb3edc3f 100644 --- a/gallery_dl/extractor/pixnet.py +++ b/gallery_dl/extractor/pixnet.py @@ -67,11 +67,7 @@ class PixnetImageExtractor(PixnetExtractor): filename_fmt = "{id}.{extension}" directory_fmt = ("{category}", "{blog}") pattern = BASE_PATTERN + r"/album/photo/(\d+)" - test = ("https://albertayu773.pixnet.net/album/photo/159443828", { - "url": "156564c422138914c9fa5b42191677b45c414af4", - "keyword": "19971bcd056dfef5593f4328a723a9602be0f087", - "content": "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", - }) + example = "https://USER.pixnet.net/album/photo/12345" def items(self): url = "https://api.pixnet.cc/oembed" @@ -99,20 +95,7 @@ class PixnetSetExtractor(PixnetExtractor): directory_fmt = ("{category}", "{blog}", "{folder_id} {folder_title}", "{set_id} {set_title}") pattern = BASE_PATTERN + r"/album/set/(\d+)" - test = ( - ("https://albertayu773.pixnet.net/album/set/15078995", { - "url": "6535712801af47af51110542f4938a7cef44557f", - "keyword": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", - }), - ("https://anrine910070.pixnet.net/album/set/5917493", { - "url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", - "keyword": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", - }), - # password-protected - ("https://sky92100.pixnet.net/album/set/17492544", { - "count": 0, - }), - ) + example = "https://USER.pixnet.net/album/set/12345" def items(self): url = self.url_fmt.format(self.root, self.item_id) @@ -157,10 +140,7 @@ class PixnetFolderExtractor(PixnetExtractor): subcategory = "folder" url_fmt = "{}/album/folder/{}" pattern = BASE_PATTERN + r"/album/folder/(\d+)" - test = ("https://albertayu773.pixnet.net/album/folder/1405768", { - "pattern": PixnetSetExtractor.pattern, - "count": ">= 15", - }) + example = "https://USER.pixnet.net/album/folder/12345" class PixnetUserExtractor(PixnetExtractor): @@ -168,16 +148,4 @@ class PixnetUserExtractor(PixnetExtractor): subcategory = "user" url_fmt = "{}{}/album/list" pattern = BASE_PATTERN + r"()(?:/blog|/album(?:/list)?)?/?(?:$|[?#])" - test = ( - ("https://albertayu773.pixnet.net/"), - ("https://albertayu773.pixnet.net/blog"), - ("https://albertayu773.pixnet.net/album"), - ("https://albertayu773.pixnet.net/album/list", { - "pattern": PixnetFolderExtractor.pattern, - "count": ">= 30", - }), - ("https://anrine910070.pixnet.net/album/list", { - "pattern": PixnetSetExtractor.pattern, - "count": ">= 14", - }), - ) + example = "https://USER.pixnet.net/" diff --git a/gallery_dl/extractor/plurk.py b/gallery_dl/extractor/plurk.py index 41352597..5a3bf5a9 100644 --- a/gallery_dl/extractor/plurk.py +++ b/gallery_dl/extractor/plurk.py @@ -71,10 +71,7 @@ class PlurkTimelineExtractor(PlurkExtractor): """Extractor for URLs from all posts in a Plurk timeline""" subcategory = "timeline" pattern = r"(?:https?://)?(?:www\.)?plurk\.com/(?!p/)(\w+)/?(?:$|[?#])" - test = ("https://www.plurk.com/plurkapi", { - "pattern": r"https?://.+", - "count": ">= 23" - }) + example = "https://www.plurk.com/USER" def __init__(self, match): PlurkExtractor.__init__(self, match) @@ -105,16 +102,7 @@ class PlurkPostExtractor(PlurkExtractor): """Extractor for URLs from a Plurk post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?plurk\.com/p/(\w+)" - test = ( - ("https://www.plurk.com/p/i701j1", { - "url": "2115f208564591b8748525c2807a84596aaaaa5f", - "count": 3, - }), - ("https://www.plurk.com/p/i701j1", { - "options": (("comments", True),), - "count": ">= 210", - }), - ) + example = "https://www.plurk.com/p/12345" def __init__(self, match): PlurkExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/poipiku.py b/gallery_dl/extractor/poipiku.py index e3bb512b..f42016fc 100644 --- a/gallery_dl/extractor/poipiku.py +++ b/gallery_dl/extractor/poipiku.py @@ -96,15 +96,7 @@ class PoipikuUserExtractor(PoipikuExtractor): subcategory = "user" pattern = (BASE_PATTERN + r"/(?:IllustListPcV\.jsp\?PG=(\d+)&ID=)?" r"(\d+)/?(?:$|[?&#])") - test = ( - ("https://poipiku.com/25049/", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049" - r"/\d+_\w+\.(jpe?g|png)$", - "range": "1-10", - "count": 10, - }), - ("https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=") - ) + example = "https://poipiku.com/12345/" def __init__(self, match): PoipikuExtractor.__init__(self, match) @@ -136,54 +128,7 @@ class PoipikuPostExtractor(PoipikuExtractor): """Extractor for a poipiku post""" subcategory = "post" pattern = BASE_PATTERN + r"/(\d+)/(\d+)" - test = ( - ("https://poipiku.com/25049/5864576.html", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049" - r"/005864576_EWN1Y65gQ\.png$", - "keyword": { - "count": "1", - "description": "", - "extension": "png", - "filename": "005864576_EWN1Y65gQ", - "num": 1, - "post_category": "DOODLE", - "post_id": "5864576", - "user_id": "25049", - "user_name": "ユキウサギ", - }, - }), - ("https://poipiku.com/2166245/6411749.html", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/002166245" - r"/006411749_\w+\.jpeg$", - "count": 4, - "keyword": { - "count": "4", - "description": "絵茶の産物ネタバレあるやつ", - "num": int, - "post_category": "SPOILER", - "post_id": "6411749", - "user_id": "2166245", - "user_name": "wadahito", - }, - }), - # different warning button style - ("https://poipiku.com/3572553/5776587.html", { - "pattern": r"https://img-org\.poipiku.com/user_img\d+/003572553" - r"/005776587_(\d+_)?\w+\.jpeg$", - "count": 3, - "keyword": { - "count": "3", - "description": "ORANGE OASISボスネタバレ<br />曲も大好き<br />" - "2枚目以降はほとんど見えなかった1枚目背景" - "のヒエログリフ小ネタです𓀀", - "num": int, - "post_category": "SPOILER", - "post_id": "5776587", - "user_id": "3572553", - "user_name": "nagakun", - }, - }), - ) + example = "https://poipiku.com/12345/12345.html" def __init__(self, match): PoipikuExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pornhub.py b/gallery_dl/extractor/pornhub.py index 6cb2063b..c5ce8327 100644 --- a/gallery_dl/extractor/pornhub.py +++ b/gallery_dl/extractor/pornhub.py @@ -56,30 +56,7 @@ class PornhubGalleryExtractor(PornhubExtractor): filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/album/(\d+)" - test = ( - ("https://www.pornhub.com/album/19289801", { - "pattern": r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", - "count": ">= 300", - "keyword": { - "id" : int, - "num" : int, - "score" : int, - "views" : int, - "caption": str, - "user" : "Danika Mori", - "gallery": { - "id" : 19289801, - "score": int, - "views": int, - "tags" : list, - "title": "Danika Mori Best Moments", - }, - }, - }), - ("https://www.pornhub.com/album/69040172", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://www.pornhub.com/album/12345" def __init__(self, match): PornhubExtractor.__init__(self, match) @@ -149,22 +126,7 @@ class PornhubGifExtractor(PornhubExtractor): filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/gif/(\d+)" - test = ( - ("https://www.pornhub.com/gif/33643461", { - "pattern": r"https://\w+\.phncdn\.com/pics/gifs" - r"/033/643/461/33643461a\.webm", - "keyword": { - "date": "dt:2020-10-31 00:00:00", - "extension": "webm", - "filename": "33643461a", - "id": "33643461", - "tags": ["big boobs", "lana rhoades"], - "title": "Big boobs", - "url": str, - "user": "Lana Rhoades", - }, - }), - ) + example = "https://www.pornhub.com/gif/12345" def __init__(self, match): PornhubExtractor.__init__(self, match) @@ -192,7 +154,7 @@ class PornhubUserExtractor(PornhubExtractor): """Extractor for a pornhub user""" subcategory = "user" pattern = BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)/?$" - test = ("https://www.pornhub.com/pornstar/danika-mori",) + example = "https://www.pornhub.com/model/USER" def __init__(self, match): PornhubExtractor.__init__(self, match) @@ -214,16 +176,7 @@ class PornhubPhotosExtractor(PornhubExtractor): subcategory = "photos" pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" "/(photos(?:/[^/?#]+)?)") - test = ( - ("https://www.pornhub.com/pornstar/danika-mori/photos", { - "pattern": PornhubGalleryExtractor.pattern, - "count": ">= 6", - }), - ("https://www.pornhub.com/users/flyings0l0/photos/public"), - ("https://www.pornhub.com/users/flyings0l0/photos/private"), - ("https://www.pornhub.com/users/flyings0l0/photos/favorites"), - ("https://www.pornhub.com/model/bossgirl/photos"), - ) + example = "https://www.pornhub.com/model/USER/photos" def __init__(self, match): PornhubExtractor.__init__(self, match) @@ -244,14 +197,7 @@ class PornhubGifsExtractor(PornhubExtractor): subcategory = "gifs" pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" "/(gifs(?:/[^/?#]+)?)") - test = ( - ("https://www.pornhub.com/pornstar/danika-mori/gifs", { - "pattern": PornhubGifExtractor.pattern, - "count": ">= 42", - }), - ("https://www.pornhub.com/users/flyings0l0/gifs"), - ("https://www.pornhub.com/model/bossgirl/gifs/video"), - ) + example = "https://www.pornhub.com/model/USER/gifs" def __init__(self, match): PornhubExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pornpics.py b/gallery_dl/extractor/pornpics.py index 929e0f58..1f14da07 100644 --- a/gallery_dl/extractor/pornpics.py +++ b/gallery_dl/extractor/pornpics.py @@ -64,33 +64,7 @@ class PornpicsExtractor(Extractor): class PornpicsGalleryExtractor(PornpicsExtractor, GalleryExtractor): """Extractor for pornpics galleries""" pattern = BASE_PATTERN + r"(/galleries/(?:[^/?#]+-)?(\d+))" - test = ( - (("https://www.pornpics.com/galleries/british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest-62610699/"), { - "pattern": r"https://cdni\.pornpics\.com/1280/7/160/62610699" - r"/62610699_\d+_[0-9a-f]{4}\.jpg", - "keyword": { - "categories": ["MILF", "Amateur", "Sexy", "Outdoor"], - "channel": "FTV MILFs", - "count": 17, - "gallery_id": 62610699, - "models": ["Danielle"], - "num": int, - "slug": "british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest", - "tags": ["Amateur MILF", "Sexy MILF"], - "title": "British beauty Danielle flashes " - "hot breasts, ass and snatch in the forest", - "views": int, - }, - }), - ("https://pornpics.com/es/galleries/62610699", { - "keyword": { - "slug": "british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest", - }, - }), - ) + example = "https://www.pornpics.com/galleries/TITLE-12345/" def __init__(self, match): PornpicsExtractor.__init__(self, match) @@ -126,14 +100,7 @@ class PornpicsTagExtractor(PornpicsExtractor): """Extractor for galleries from pornpics tag searches""" subcategory = "tag" pattern = BASE_PATTERN + r"/tags/([^/?#]+)" - test = ( - ("https://www.pornpics.com/tags/summer-dress/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://pornpics.com/fr/tags/summer-dress"), - ) + example = "https://www.pornpics.com/tags/TAGS/" def galleries(self): url = "{}/tags/{}/".format(self.root, self.item) @@ -144,26 +111,7 @@ class PornpicsSearchExtractor(PornpicsExtractor): """Extractor for galleries from pornpics search results""" subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?q=|pornstars/|channels/)([^/&#]+)" - test = ( - ("https://www.pornpics.com/?q=nature", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://www.pornpics.com/channels/femjoy/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://www.pornpics.com/pornstars/emma-brown/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://pornpics.com/jp/?q=nature"), - ("https://pornpics.com/it/channels/femjoy"), - ("https://pornpics.com/pt/pornstars/emma-brown"), - ) + example = "https://www.pornpics.com/?q=QUERY" def galleries(self): url = self.root + "/search/srch.php" diff --git a/gallery_dl/extractor/pururin.py b/gallery_dl/extractor/pururin.py index e56dbcd6..3a4c614b 100644 --- a/gallery_dl/extractor/pururin.py +++ b/gallery_dl/extractor/pururin.py @@ -15,37 +15,9 @@ from .. import text, util class PururinGalleryExtractor(GalleryExtractor): """Extractor for image galleries on pururin.io""" category = "pururin" - pattern = r"(?:https?://)?(?:www\.)?pururin\.[ti]o/(?:gallery|read)/(\d+)" - test = ( - ("https://pururin.to/gallery/38661/iowant-2", { - "pattern": r"https://i\.pururin\.to/38661/\d+\.jpg", - "keyword": { - "title" : "re:I ?owant 2!!", - "title_en" : "re:I ?owant 2!!", - "title_ja" : "", - "gallery_id": 38661, - "count" : 19, - "artist" : ["Shoda Norihiro"], - "group" : ["Obsidian Order"], - "parody" : ["Kantai Collection"], - "characters": ["Iowa", "Teitoku"], - "tags" : list, - "type" : "Doujinshi", - "collection": ["I owant you!"], - "convention": ["C92"], - "rating" : float, - "uploader" : "demo", - "scanlator" : ["mrwayne", "The Lost Light"], - "lang" : "en", - "language" : "English", - } - }), - ("https://pururin.to/gallery/7661/unisis-team-vanilla", { - "count": 17, - }), - ("https://pururin.io/gallery/38661/iowant-2"), - ) root = "https://pururin.to" + pattern = r"(?:https?://)?(?:www\.)?pururin\.[ti]o/(?:gallery|read)/(\d+)" + example = "https://pururin.to/gallery/12345/TITLE" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index d10c412b..abe993a6 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -173,25 +173,7 @@ class ReactorTagExtractor(ReactorExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "{search_tags}_{post_id}_{num}" pattern = BASE_PATTERN + r"/tag/([^/?#]+)(?:/[^/?#]+)?" - test = ( - ("http://reactor.cc/tag/gif"), - ("http://anime.reactor.cc/tag/Anime+Art"), - ("http://joyreactor.cc/tag/Advent+Cirno", { - "count": ">= 15", - }), - ("http://joyreactor.com/tag/Cirno", { - "url": "aa59090590b26f4654881301fe8fe748a51625a8", - }), - # 'best' rating (#3073) - ("http://joyreactor.com/tag/Dark+Souls+2/best", { - "count": 4, - }), - ("http://pornreactor.cc/tag/RiceGnat", { - "range": "1-25", - "count": ">= 25", - }), - ("http://fapreactor.com/tag/RiceGnat"), - ) + example = "http://reactor.cc/tag/TAG" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -207,19 +189,7 @@ class ReactorSearchExtractor(ReactorExtractor): directory_fmt = ("{category}", "search", "{search_tags}") archive_fmt = "s_{search_tags}_{post_id}_{num}" pattern = BASE_PATTERN + r"/search(?:/|\?q=)([^/?#]+)" - test = ( - ("http://reactor.cc/search?q=Art"), - ("http://joyreactor.cc/search/Nature", { - "range": "1-25", - "count": ">= 20", - }), - ("http://joyreactor.com/search?q=Nature", { - "range": "1-25", - "count": ">= 20", - }), - ("http://pornreactor.cc/search?q=ecchi+hentai"), - ("http://fapreactor.com/search/ecchi+hentai"), - ) + example = "http://reactor.cc/search?q=QUERY" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -234,19 +204,7 @@ class ReactorUserExtractor(ReactorExtractor): subcategory = "user" directory_fmt = ("{category}", "user", "{user}") pattern = BASE_PATTERN + r"/user/([^/?#]+)" - test = ( - ("http://reactor.cc/user/Dioklet"), - ("http://anime.reactor.cc/user/Shuster"), - ("http://joyreactor.cc/user/hemantic"), - ("http://joyreactor.com/user/Tacoman123", { - "url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", - }), - ("http://pornreactor.cc/user/Disillusion", { - "range": "1-25", - "count": ">= 20", - }), - ("http://fapreactor.com/user/Disillusion"), - ) + example = "http://reactor.cc/user/USER" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -260,47 +218,7 @@ class ReactorPostExtractor(ReactorExtractor): """Extractor for single posts on *reactor.cc sites""" subcategory = "post" pattern = BASE_PATTERN + r"/post/(\d+)" - test = ( - ("http://reactor.cc/post/4999736", { - "url": "dfc74d150d7267384d8c229c4b82aa210755daa0", - }), - ("http://anime.reactor.cc/post/3576250"), - # single image - ("http://joyreactor.com/post/3721876", { - "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" - r"/cartoon-painting-monster-lake-4841316.jpeg", - "count": 1, - "keyword": "2207a7dfed55def2042b6c2554894c8d7fda386e", - }), - # 4 images - ("http://joyreactor.com/post/3713804", { - "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" - r"/movie-tv-godzilla-monsters-\d+\.jpeg", - "count": 4, - "keyword": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", - }), - # gif / video - ("http://joyreactor.com/post/3726210", { - "url": "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", - "keyword": "8949d9d5fc469dab264752432efbaa499561664a", - }), - # youtube embed - ("http://joyreactor.com/post/3668724", { - "url": "bf1666eddcff10c9b58f6be63fa94e4e13074214", - "keyword": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", - }), - # "malformed" JSON - ("http://joyreactor.cc/post/1299", { - "url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", - }), - ("http://pornreactor.cc/post/863166", { - "url": "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", - "content": "ec6b0568bfb1803648744077da082d14de844340", - }), - ("http://fapreactor.com/post/863166", { - "url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", - }), - ) + example = "http://reactor.cc/post/12345" def __init__(self, match): ReactorExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/readcomiconline.py b/gallery_dl/extractor/readcomiconline.py index c68068cb..93e41be1 100644 --- a/gallery_dl/extractor/readcomiconline.py +++ b/gallery_dl/extractor/readcomiconline.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -49,11 +49,7 @@ class ReadcomiconlineIssueExtractor(ReadcomiconlineBase, ChapterExtractor): """Extractor for comic-issues from readcomiconline.li""" subcategory = "issue" pattern = BASE_PATTERN + r"(/Comic/[^/?#]+/[^/?#]+\?)([^#]+)" - test = ("https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", { - "pattern": r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", - "keyword": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", - "count": 36, - }) + example = "https://readcomiconline.li/Comic/TITLE/Issue-123?id=12345" def __init__(self, match): ChapterExtractor.__init__(self, match) @@ -98,16 +94,7 @@ class ReadcomiconlineComicExtractor(ReadcomiconlineBase, MangaExtractor): chapterclass = ReadcomiconlineIssueExtractor subcategory = "comic" pattern = BASE_PATTERN + r"(/Comic/[^/?#]+/?)$" - test = ( - ("https://readcomiconline.li/Comic/W-i-t-c-h", { - "url": "74eb8b9504b4084fcc9367b341300b2c52260918", - "keyword": "3986248e4458fa44a201ec073c3684917f48ee0c", - }), - ("https://readcomiconline.to/Comic/Bazooka-Jules", { - "url": "2f66a467a772df4d4592e97a059ddbc3e8991799", - "keyword": "f5ba5246cd787bb750924d9690cb1549199bd516", - }), - ) + example = "https://readcomiconline.li/Comic/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/recursive.py b/gallery_dl/extractor/recursive.py index 4dd9d5c0..da1f5eac 100644 --- a/gallery_dl/extractor/recursive.py +++ b/gallery_dl/extractor/recursive.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2020 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -17,9 +17,7 @@ class RecursiveExtractor(Extractor): """Extractor that fetches URLs from a remote or local source""" category = "recursive" pattern = r"r(?:ecursive)?:" - test = ("recursive:https://pastebin.com/raw/FLwrCYsT", { - "url": "eee86d65c346361b818e8f4b2b307d9429f136a2", - }) + example = "recursive:https://pastebin.com/raw/FLwrCYsT" def items(self): self.session.mount("file://", FileAdapter()) diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 36708f1a..c64576f3 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -195,16 +195,7 @@ class RedditSubredditExtractor(RedditExtractor): subcategory = "subreddit" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com" r"(/r/[^/?#]+(?:/([a-z]+))?)/?(?:\?([^#]*))?(?:$|#)") - test = ( - ("https://www.reddit.com/r/lavaporn/", { - "range": "1-20", - "count": ">= 20", - }), - ("https://www.reddit.com/r/lavaporn/top/?sort=top&t=month"), - ("https://old.reddit.com/r/lavaporn/"), - ("https://np.reddit.com/r/lavaporn/"), - ("https://m.reddit.com/r/lavaporn/"), - ) + example = "https://www.reddit.com/r/SUBREDDIT/" def __init__(self, match): self.subreddit, sub, params = match.groups() @@ -222,13 +213,7 @@ class RedditHomeExtractor(RedditSubredditExtractor): subcategory = "home" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com" r"((?:/([a-z]+))?)/?(?:\?([^#]*))?(?:$|#)") - test = ( - ("https://www.reddit.com/", { - "range": "1-20", - "count": ">= 20", - }), - ("https://old.reddit.com/top/?sort=top&t=month"), - ) + example = "https://www.reddit.com/" class RedditUserExtractor(RedditExtractor): @@ -236,14 +221,7 @@ class RedditUserExtractor(RedditExtractor): subcategory = "user" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com/u(?:ser)?/" r"([^/?#]+(?:/([a-z]+))?)/?(?:\?([^#]*))?$") - test = ( - ("https://www.reddit.com/user/username/", { - "count": ">= 2", - }), - ("https://www.reddit.com/user/username/gilded/?sort=top&t=month"), - ("https://old.reddit.com/user/username/"), - ("https://www.reddit.com/u/username/"), - ) + example = "https://www.reddit.com/user/USER/" def __init__(self, match): self.user, sub, params = match.groups() @@ -262,76 +240,7 @@ class RedditSubmissionExtractor(RedditExtractor): pattern = (r"(?:https?://)?(?:" r"(?:\w+\.)?reddit\.com/(?:(?:r|u|user)/[^/?#]+" r"/comments|gallery)|redd\.it)/([a-z0-9]+)") - test = ( - ("https://www.reddit.com/r/lavaporn/comments/8cqhub/", { - "pattern": r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", - "count": 1, - }), - ("https://www.reddit.com/r/lavaporn/comments/8cqhub/", { - "options": (("comments", 500),), - "pattern": r"https://", - "count": 3, - }), - ("https://www.reddit.com/gallery/hrrh23", { - "url": "25b91ede15459470274dd17291424b037ed8b0ae", - "content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", - "count": 3, - }), - # video - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11", - "count": 1, - }), - # video (ytdl) - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "options": (("videos", "ytdl"),), - "pattern": r"ytdl:https://www.reddit.com/r/aww/comments/90bu6w" - r"/heat_index_was_110_degrees_so_we_offered_him_a/", - "count": 1, - }), - # video (dash) - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "options": (("videos", "dash"),), - "pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11" - r"/DASHPlaylist.mpd\?a=", - "count": 1, - }), - # deleted gallery (#953) - ("https://www.reddit.com/gallery/icfgzv", { - "count": 0, - }), - # animated gallery items (#955) - ("https://www.reddit.com/r/araragi/comments/ib32hm", { - "pattern": r"https://i\.redd\.it/\w+\.gif", - "count": 2, - }), - # "failed" gallery item (#1127) - ("https://www.reddit.com/r/cosplay/comments/jvwaqr", { - "count": 1, - }), - # gallery with no 'media_metadata' (#2001) - ("https://www.reddit.com/r/kpopfap/comments/qjj04q/", { - "count": 0, - }), - # user page submission (#2301) - ("https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", { - "pattern": r"https://i.redd.it/8fpgv17yqlh81.jpg", - "count": 1, - }), - # cross-posted video (#887, #3586, #3976) - ("https://www.reddit.com/r/kittengifs/comments/12m0b8d", { - "pattern": r"ytdl:https://v\.redd\.it/cvabpjacrvta1", - }), - # preview.redd.it (#4470) - ("https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", { - "pattern": r"https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp" - r"&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", - }), - ("https://old.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://np.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://m.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://redd.it/2a00np/"), - ) + example = "https://www.reddit.com/r/SUBREDDIT/comments/ID/" def __init__(self, match): RedditExtractor.__init__(self, match) @@ -348,22 +257,7 @@ class RedditImageExtractor(Extractor): archive_fmt = "{filename}" pattern = (r"(?:https?://)?((?:i|preview)\.redd\.it|i\.reddituploads\.com)" r"/([^/?#]+)(\?[^#]*)?") - test = ( - ("https://i.redd.it/upjtjcx2npzz.jpg", { - "url": "0de614900feef103e580b632190458c0b62b641a", - "content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", - }), - (("https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d" - "?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2"), { - "url": "f24f25efcedaddeec802e46c60d77ef975dc52a5", - "content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", - }), - # preview.redd.it -> i.redd.it - (("https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart" - "&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac"), { - "pattern": r"^https://i\.redd\.it/00af44lpn0u51\.jpg$" - }), - ) + example = "https://i.redd.it/NAME.EXT" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index abd21b30..69a30b67 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -90,16 +90,7 @@ class RedgifsUserExtractor(RedgifsExtractor): subcategory = "user" directory_fmt = ("{category}", "{userName}") pattern = r"(?:https?://)?(?:\w+\.)?redgifs\.com/users/([^/?#]+)/?$" - test = ( - ("https://www.redgifs.com/users/Natalifiction", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "count": ">= 100", - }), - ("https://v3.redgifs.com/users/lamsinka89", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "count": ">= 100", - }), - ) + example = "https://www.redgifs.com/users/USER" def metadata(self): return {"userName": self.key} @@ -115,18 +106,7 @@ class RedgifsCollectionExtractor(RedgifsExtractor): archive_fmt = "{folderId}_{id}" pattern = (r"(?:https?://)?(?:www\.)?redgifs\.com/users" r"/([^/?#]+)/collections/([^/?#]+)") - test = ( - ("https://www.redgifs.com/users/boombah123/collections/2631326bbd", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ("https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ) + example = "https://www.redgifs.com/users/USER/collections/ID" def __init__(self, match): RedgifsExtractor.__init__(self, match) @@ -146,11 +126,7 @@ class RedgifsCollectionsExtractor(RedgifsExtractor): subcategory = "collections" pattern = (r"(?:https?://)?(?:www\.)?redgifs\.com/users" r"/([^/?#]+)/collections/?$") - test = ("https://www.redgifs.com/users/boombah123/collections", { - "pattern": (r"https://www\.redgifs\.com/users" - r"/boombah123/collections/\w+"), - "count": ">= 3", - }) + example = "https://www.redgifs.com/users/USER/collections" def items(self): for collection in self.api.collections(self.key): @@ -164,18 +140,7 @@ class RedgifsNichesExtractor(RedgifsExtractor): """Extractor for redgifs niches""" subcategory = "niches" pattern = r"(?:https?://)?(?:www\.)?redgifs\.com/niches/([^/?#]+)" - test = ( - ("https://www.redgifs.com/niches/boobs", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ("https://www.redgifs.com/niches/ass", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ) + example = "https://www.redgifs.com/niches/NAME" def gifs(self): return self.api.niches(self.key) @@ -187,21 +152,7 @@ class RedgifsSearchExtractor(RedgifsExtractor): directory_fmt = ("{category}", "Search", "{search}") pattern = (r"(?:https?://)?(?:\w+\.)?redgifs\.com" r"/(?:gifs/([^/?#]+)|browse)(?:/?\?([^#]+))?") - test = ( - ("https://www.redgifs.com/gifs/jav", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "range": "1-10", - "count": 10, - }), - ("https://www.redgifs.com/browse?tags=JAV", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "range": "1-10", - "count": 10, - }), - ("https://www.redgifs.com/gifs/jav?order=best&verified=1"), - ("https://www.redgifs.com/browse?type=i&verified=y&order=top7"), - ("https://v3.redgifs.com/browse?tags=JAV"), - ) + example = "https://www.redgifs.com/gifs/TAG" def __init__(self, match): RedgifsExtractor.__init__(self, match) @@ -227,27 +178,7 @@ class RedgifsImageExtractor(RedgifsExtractor): r"(?:\w+\.)?redgifs\.com/(?:watch|ifr)|" r"(?:www\.)?gifdeliverynetwork\.com|" r"i\.redgifs\.com/i)/([A-Za-z]+)") - test = ( - ("https://redgifs.com/watch/foolishforkedabyssiniancat", { - "pattern": r"https://\w+\.redgifs\.com" - r"/FoolishForkedAbyssiniancat\.mp4", - "content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", - }), - # gallery (#4021) - ("https://www.redgifs.com/watch/desertedbaregraywolf", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", - "count": 4, - "keyword": { - "num": int, - "count": 4, - "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", - }, - }), - ("https://redgifs.com/ifr/FoolishForkedAbyssiniancat"), - ("https://i.redgifs.com/i/FoolishForkedAbyssiniancat"), - ("https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat"), - ("https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat"), - ) + example = "https://redgifs.com/watch/ID" def gifs(self): return (self.api.gif(self.key),) diff --git a/gallery_dl/extractor/rule34us.py b/gallery_dl/extractor/rule34us.py index 88331eaa..6439a225 100644 --- a/gallery_dl/extractor/rule34us.py +++ b/gallery_dl/extractor/rule34us.py @@ -10,8 +10,8 @@ from .booru import BooruExtractor from .. import text -import re import collections +import re class Rule34usExtractor(BooruExtractor): @@ -54,11 +54,7 @@ class Rule34usTagExtractor(Rule34usExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = r"(?:https?://)?rule34\.us/index\.php\?r=posts/index&q=([^&#]+)" - test = ("https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", { - "pattern": r"https://img\d*\.rule34\.us" - r"/images/../../[0-9a-f]{32}\.\w+", - "count": 10, - }) + example = "https://rule34.us/index.php?r=posts/index&q=TAG" def __init__(self, match): Rule34usExtractor.__init__(self, match) @@ -95,31 +91,7 @@ class Rule34usPostExtractor(Rule34usExtractor): subcategory = "post" archive_fmt = "{id}" pattern = r"(?:https?://)?rule34\.us/index\.php\?r=posts/view&id=(\d+)" - test = ( - ("https://rule34.us/index.php?r=posts/view&id=3709005", { - "pattern": r"https://img\d*\.rule34\.us/images/14/7b" - r"/147bee6fc2e13f73f5f9bac9d4930b13\.png", - "content": "d714342ea84050f82dda5f0c194d677337abafc5", - }), - ("https://rule34.us/index.php?r=posts/view&id=4576310", { - "pattern": r"https://video\.rule34\.us/images/a2/94" - r"/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", - "keyword": { - "extension": "mp4", - "file_url": str, - "filename": "a294ff8e1f8e0efa041e5dc9d1480011", - "height": "3982", - "id": "4576310", - "md5": "a294ff8e1f8e0efa041e5dc9d1480011", - "score": r"re:\d+", - "tags": "tagme, video", - "tags_general": "video", - "tags_metadata": "tagme", - "uploader": "Anonymous", - "width": "3184", - }, - }), - ) + example = "https://rule34.us/index.php?r=posts/view&id=12345" def __init__(self, match): Rule34usExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index 303a8e55..f9fe7fa3 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -88,32 +88,7 @@ class SankakuTagExtractor(SankakuExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/?\?([^#]*)" - test = ( - ("https://sankaku.app/?tags=bonocho", { - "count": 5, - "pattern": r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" - r"/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", - }), - ("https://beta.sankakucomplex.com/?tags=bonocho"), - ("https://chan.sankakucomplex.com/?tags=bonocho"), - ("https://black.sankakucomplex.com/?tags=bonocho"), - ("https://white.sankakucomplex.com/?tags=bonocho"), - ("https://sankaku.app/ja?tags=order%3Apopularity"), - ("https://sankaku.app/no/?tags=order%3Apopularity"), - # error on five or more tags - ("https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", { - "options": (("username", None),), - "exception": exception.StopExtraction, - }), - # match arbitrary query parameters - ("https://chan.sankakucomplex.com" - "/?tags=marie_rose&page=98&next=3874906&commit=Search"), - # 'date:' tags (#1790) - ("https://chan.sankakucomplex.com/?tags=date:2023-03-20", { - "range": "1", - "count": 1, - }), - ) + example = "https://sankaku.app/?tags=TAG" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -143,13 +118,7 @@ class SankakuPoolExtractor(SankakuExtractor): directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name_en]}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/(?:books|pool/show)/(\d+)" - test = ( - ("https://sankaku.app/books/90", { - "count": 5, - }), - ("https://beta.sankakucomplex.com/books/90"), - ("https://chan.sankakucomplex.com/pool/show/90"), - ) + example = "https://sankaku.app/books/12345" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -175,50 +144,7 @@ class SankakuPostExtractor(SankakuExtractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post/show/([0-9a-f]+)" - test = ( - ("https://sankaku.app/post/show/360451", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "options": (("tags", True),), - "keyword": { - "tags_artist" : ["bonocho"], - "tags_studio" : ["dc_comics"], - "tags_medium" : list, - "tags_copyright": list, - "tags_character": list, - "tags_general" : list, - }, - }), - # 'contentious_content' - ("https://sankaku.app/post/show/21418978", { - "pattern": r"https://s\.sankakucomplex\.com" - r"/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", - }), - # empty tags (#1617) - ("https://sankaku.app/post/show/20758561", { - "options": (("tags", True),), - "count": 1, - "keyword": { - "tags": list, - "tags_general": ["key(mangaka)", "key(mangaka)"], - }, - }), - # md5 hexdigest instead of ID (#3952) - (("https://chan.sankakucomplex.com/post/show" - "/f8ba89043078f0e4be2d9c46550b840a"), { - "pattern": r"https://s\.sankakucomplex\.com" - r"/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "count": 1, - "keyword": { - "id": 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", - }, - }), - ("https://chan.sankakucomplex.com/post/show/360451"), - ("https://chan.sankakucomplex.com/ja/post/show/360451"), - ("https://beta.sankakucomplex.com/post/show/360451"), - ("https://white.sankakucomplex.com/post/show/360451"), - ("https://black.sankakucomplex.com/post/show/360451"), - ) + example = "https://sankaku.app/post/show/12345" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -232,13 +158,7 @@ class SankakuBooksExtractor(SankakuExtractor): """Extractor for books by tag search on sankaku.app""" subcategory = "books" pattern = BASE_PATTERN + r"/books/?\?([^#]*)" - test = ( - ("https://sankaku.app/books?tags=aiue_oka", { - "range": "1-20", - "count": 20, - }), - ("https://beta.sankakucomplex.com/books?tags=aiue_oka"), - ) + example = "https://sankaku.app/books?tags=TAG" def __init__(self, match): SankakuExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/sankakucomplex.py b/gallery_dl/extractor/sankakucomplex.py index 47e067b2..e1d4153a 100644 --- a/gallery_dl/extractor/sankakucomplex.py +++ b/gallery_dl/extractor/sankakucomplex.py @@ -30,32 +30,8 @@ class SankakucomplexArticleExtractor(SankakucomplexExtractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{date:%Y%m%d}_{filename}" pattern = (r"(?:https?://)?www\.sankakucomplex\.com" - r"/(\d{4}/\d\d/\d\d/[^/?#]+)") - test = ( - ("https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", { - "url": "4a9ecc5ae917fbce469280da5b6a482510cae84d", - "keyword": "bfe08310e7d9a572f568f6900e0ed0eb295aa2b3", - }), - ("https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", { - "url": "a1e249173fd6c899a8134fcfbd9c925588a63f7c", - "keyword": "e78fcc23c2711befc0969a45ea5082a29efccf68", - }), - # videos (#308) - (("https://www.sankakucomplex.com/2019/06/11" - "/darling-ol-goddess-shows-off-her-plump-lower-area/"), { - "pattern": r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", - "range": "26-", - "count": 5, - }), - # youtube embeds (#308) - (("https://www.sankakucomplex.com/2015/02/12" - "/snow-miku-2015-live-magical-indeed/"), { - "options": (("embeds", True),), - "pattern": r"https://www.youtube.com/embed/", - "range": "2-", - "count": 2, - }), - ) + r"/(\d\d\d\d/\d\d/\d\d/[^/?#]+)") + example = "https://www.sankakucomplex.com/1970/01/01/TITLE" def items(self): url = "{}/{}/?pg=X".format(self.root, self.path) @@ -113,15 +89,7 @@ class SankakucomplexTagExtractor(SankakucomplexExtractor): subcategory = "tag" pattern = (r"(?:https?://)?www\.sankakucomplex\.com" r"/((?:tag|category|author)/[^/?#]+)") - test = ( - ("https://www.sankakucomplex.com/tag/cosplay/", { - "range": "1-50", - "count": 50, - "pattern": SankakucomplexArticleExtractor.pattern, - }), - ("https://www.sankakucomplex.com/category/anime/"), - ("https://www.sankakucomplex.com/author/rift/page/5/"), - ) + example = "https://www.sankakucomplex.com/tag/TAG/" def items(self): pnum = 1 diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index ab466149..edfe1dcd 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -58,32 +58,7 @@ class SeigaUserExtractor(SeigaExtractor): filename_fmt = "{category}_{user[id]}_{image_id}.{extension}" pattern = (r"(?:https?://)?(?:www\.|(?:sp\.)?seiga\.)?nicovideo\.jp/" r"user/illust/(\d+)(?:\?(?:[^&]+&)*sort=([^&#]+))?") - test = ( - ("https://seiga.nicovideo.jp/user/illust/39537793", { - "pattern": r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", - "count": ">= 4", - "keyword": { - "user": { - "id": 39537793, - "message": str, - "name": str, - }, - "clips": int, - "comments": int, - "count": int, - "extension": None, - "image_id": int, - "title": str, - "views": int, - }, - }), - ("https://seiga.nicovideo.jp/user/illust/79433", { - "exception": exception.NotFoundError, - }), - ("https://seiga.nicovideo.jp/user/illust/39537793" - "?sort=image_view&target=illust_all"), - ("https://sp.seiga.nicovideo.jp/user/illust/39537793"), - ) + example = "https://seiga.nicovideo.jp/user/illust/12345" def __init__(self, match): SeigaExtractor.__init__(self, match) @@ -156,26 +131,7 @@ class SeigaImageExtractor(SeigaExtractor): r"(?:seiga\.|www\.)?nicovideo\.jp/(?:seiga/im|image/source/)" r"|sp\.seiga\.nicovideo\.jp/seiga/#!/im" r"|lohas\.nicoseiga\.jp/(?:thumb|(?:priv|o)/[^/]+/\d+)/)(\d+)") - test = ( - ("https://seiga.nicovideo.jp/seiga/im5977527", { - "keyword": "c8339781da260f7fc44894ad9ada016f53e3b12a", - "content": "d9202292012178374d57fb0126f6124387265297", - }), - ("https://seiga.nicovideo.jp/seiga/im123", { - "exception": exception.NotFoundError, - }), - ("https://seiga.nicovideo.jp/seiga/im10877923", { - "pattern": r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e46" - r"5e0411c0822e0b510e286/1688757110/10877923", - }), - ("https://seiga.nicovideo.jp/image/source/5977527"), - ("https://sp.seiga.nicovideo.jp/seiga/#!/im5977527"), - ("https://lohas.nicoseiga.jp/thumb/5977527i"), - ("https://lohas.nicoseiga.jp/priv" - "/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527"), - ("https://lohas.nicoseiga.jp/o" - "/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527"), - ) + example = "https://seiga.nicovideo.jp/seiga/im12345" def __init__(self, match): SeigaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/senmanga.py b/gallery_dl/extractor/senmanga.py index c48be84e..a8fdc4c9 100644 --- a/gallery_dl/extractor/senmanga.py +++ b/gallery_dl/extractor/senmanga.py @@ -17,54 +17,7 @@ class SenmangaChapterExtractor(ChapterExtractor): category = "senmanga" root = "https://raw.senmanga.com" pattern = r"(?:https?://)?raw\.senmanga\.com(/[^/?#]+/[^/?#]+)" - test = ( - ("https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", { - "pattern": r"https://raw\.senmanga\.com/viewer" - r"/Bokura-wa-Minna-Kawaisou/37A/[12]", - "url": "5f95140ff511d8497e2ec08fa7267c6bb231faec", - "content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", - "keyword": { - "chapter": "37A", - "count": 2, - "extension": "", - "filename": "re:[12]", - "lang": "ja", - "language": "Japanese", - "manga": "Bokura wa Minna Kawaisou", - "page": int, - }, - }), - ("http://raw.senmanga.com/Love-Lab/2016-03/1", { - "pattern": r"https://raw\.senmanga\.com/viewer" - r"/Love-Lab/2016-03/\d", - "url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", - "keyword": { - "chapter": "2016-03", - "count": 9, - "extension": "", - "filename": r"re:\d", - "manga": "Renai Lab 恋愛ラボ", - }, - }), - ("https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", { - "pattern": r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a" - r"/akabane-honeko-no-bodyguard/chapter-1" - r"/\d+-[0-9a-f]{13}\.jpg", - "keyword": { - "chapter": "1", - "count": 65, - "extension": "jpg", - "filename": r"re:\d+-\w+", - "manga": "Akabane Honeko no Bodyguard", - }, - }), - # no http scheme () - ("https://raw.senmanga.com/amama-cinderella/3", { - "pattern": r"^https://kumacdn.club/image-new-2/a/amama-cinderella" - r"/chapter-3/.+\.jpg", - "count": 30, - }), - ) + example = "https://raw.senmanga.com/MANGA/CHAPTER" def _init(self): self.session.headers["Referer"] = self.gallery_url diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py index c6588def..80f2aea5 100644 --- a/gallery_dl/extractor/sexcom.py +++ b/gallery_dl/extractor/sexcom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -106,43 +106,7 @@ class SexcomPinExtractor(SexcomExtractor): subcategory = "pin" directory_fmt = ("{category}",) pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+)(?!.*#related$)" - test = ( - # picture - ("https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", { - "pattern": "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", - "content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", - "keyword": { - "comments" : int, - "date" : "dt:2014-10-19 15:45:44", - "extension": "jpg", - "filename" : "7637609", - "likes" : int, - "pin_id" : 21241874, - "repins" : int, - "tags" : list, - "thumbnail": str, - "title" : "Sexy Ecchi Girls 166", - "type" : "picture", - "uploader" : "mangazeta", - "url" : str, - }, - }), - # gif - ("https://www.sex.com/pin/55435122-ecchi/", { - "pattern": "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", - "content": "176cc63fa05182cb0438c648230c0f324a5965fe", - }), - # video - ("https://www.sex.com/pin/55748341/", { - "pattern": r"https://cdn\.sex\.com/videos/pinporn" - r"/2018/02/10/776229_hd\.mp4", - "content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", - }), - # pornhub embed - ("https://www.sex.com/pin/55847384-very-nicely-animated/", { - "pattern": "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", - }), - ) + example = "https://www.sex.com/pin/12345-TITLE/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -157,9 +121,7 @@ class SexcomRelatedPinExtractor(SexcomPinExtractor): subcategory = "related-pin" directory_fmt = ("{category}", "related {original_pin[pin_id]}") pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+).*#related$" - test = ("https://www.sex.com/pin/21241874/#related", { - "count": ">= 20", - }) + example = "https://www.sex.com/pin/12345#related" def metadata(self): pin = self._parse_pin(SexcomPinExtractor.pins(self)[0]) @@ -176,9 +138,7 @@ class SexcomPinsExtractor(SexcomExtractor): subcategory = "pins" directory_fmt = ("{category}", "{user}") pattern = r"(?:https?://)?(?:www\.)?sex\.com/user/([^/?#]+)/pins/" - test = ("https://www.sex.com/user/sirjuan79/pins/", { - "count": ">= 15", - }) + example = "https://www.sex.com/user/USER/pins/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -198,9 +158,7 @@ class SexcomBoardExtractor(SexcomExtractor): directory_fmt = ("{category}", "{user}", "{board}") pattern = (r"(?:https?://)?(?:www\.)?sex\.com/user" r"/([^/?#]+)/(?!(?:following|pins|repins|likes)/)([^/?#]+)") - test = ("https://www.sex.com/user/ronin17/exciting-hentai/", { - "count": ">= 15", - }) + example = "https://www.sex.com/user/USER/BOARD/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -224,17 +182,7 @@ class SexcomSearchExtractor(SexcomExtractor): pattern = (r"(?:https?://)?(?:www\.)?sex\.com/((?:" r"(pic|gif|video)s/([^/?#]*)|search/(pic|gif|video)s" r")/?(?:\?([^#]+))?)") - test = ( - ("https://www.sex.com/search/pics?query=ecchi", { - "range": "1-10", - "count": 10, - }), - ("https://www.sex.com/videos/hentai/", { - "range": "1-10", - "count": 10, - }), - ("https://www.sex.com/pics/?sort=popular&sub=all&page=1"), - ) + example = "https://www.sex.com/search/pics?query=QUERY" def __init__(self, match): SexcomExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/shimmie2.py b/gallery_dl/extractor/shimmie2.py index 4ae4e96e..99dc9034 100644 --- a/gallery_dl/extractor/shimmie2.py +++ b/gallery_dl/extractor/shimmie2.py @@ -100,36 +100,7 @@ class Shimmie2TagExtractor(Shimmie2Extractor): directory_fmt = ("{category}", "{search_tags}") file_url_fmt = "{}/_images/{}/{}%20-%20{}.{}" pattern = BASE_PATTERN + r"post/list/([^/?#]+)(?:/(\d+))?()" - test = ( - ("https://meme.museum/post/list/animated/1", { - "pattern": r"https://meme\.museum/_images/\w+/\d+%20-%20", - "count": ">= 30" - }), - ("https://loudbooru.com/post/list/original_character/1", { - "pattern": r"https://loudbooru\.com/_images/[0-9a-f]{32}/\d+", - "range": "1-100", - "count": 100, - }), - ("https://giantessbooru.com/index.php?q=/post/list/drawing/1", { - "pattern": r"https://giantessbooru\.com/index\.php" - r"\?q=/image/\d+\.jpg", - "range": "1-100", - "count": 100, - }), - ("https://giantessbooru.com/post/list/drawing/1"), - ("https://tentaclerape.net/post/list/comic/1", { - "pattern": r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", - "range": "1-100", - "count": 100, - }), - ("https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", { - "pattern": r"https://booru\.cavemanon\.xyz" - r"/index\.php\?q=image/\d+\.\w+", - "range": "1-100", - "count": 100, - }), - ("https://booru.cavemanon.xyz/post/list/Amber/1"), - ) + example = "https://loudbooru.com/post/list/TAG/1" def __init__(self, match): Shimmie2Extractor.__init__(self, match) @@ -225,115 +196,7 @@ class Shimmie2PostExtractor(Shimmie2Extractor): """Extractor for single shimmie2 posts""" subcategory = "post" pattern = BASE_PATTERN + r"post/view/(\d+)" - test = ( - ("https://meme.museum/post/view/10243", { - "pattern": r"https://meme\.museum/_images/105febebcd5ca791ee332adc" - r"49971f78/10243%20-%20g%20beard%20open_source%20richar" - r"d_stallman%20stallman%20tagme%20text\.jpg", - "content": "45565f3f141fc960a8ae1168b80e718a494c52d2", - "keyword": { - "extension": "jpg", - "file_url": "https://meme.museum/_images/105febebcd5ca791ee332" - "adc49971f78/10243%20-%20g%20beard%20open_source%2" - "0richard_stallman%20stallman%20tagme%20text.jpg", - "filename": "10243 - g beard open_source richard_stallman " - "stallman tagme text", - "height": 451, - "id": 10243, - "md5": "105febebcd5ca791ee332adc49971f78", - "size": 0, - "subcategory": "post", - "tags": "/g/ beard open_source " - "richard_stallman stallman tagme text", - "width": 480, - }, - }), - ("https://loudbooru.com/post/view/33828", { - "pattern": r"https://loudbooru\.com/_images/.+\.png", - "content": "a4755f787ba23ae2aa297a46810f802ca9032739", - "keyword": { - "extension": "png", - "file_url": "https://loudbooru.com/_images/ca2638d903c86e8337f" - "e9aeb4974be88/33828%20-%202020%20artist%3Astikyfi" - "nkaz%20character%3Alisa_loud%20cover%20fanfiction" - "%3Aplatz_eins%20frowning%20half-closed_eyes%20sol" - "o%20text%20title_card.png", - "filename": "33828 - 2020 artist:stikyfinkaz character:lisa_" - "loud cover fanfiction:platz_eins frowning " - "half-closed_eyes solo text title_card", - "height": 1920, - "id": 33828, - "md5": "ca2638d903c86e8337fe9aeb4974be88", - "tags": "2020 artist:stikyfinkaz character:lisa_loud cover " - "fanfiction:platz_eins frowning half-closed_eyes " - "solo text title_card", - "width": 1078, - }, - }), - ("https://giantessbooru.com/index.php?q=/post/view/41", { - "pattern": r"https://giantessbooru\.com/index\.php" - r"\?q=/image/41\.jpg", - "content": "79115ed309d1f4e82e7bead6948760e889139c91", - "keyword": { - "extension": "jpg", - "file_url": "https://giantessbooru.com/index.php" - "?q=/image/41.jpg", - "filename": "41", - "height": 0, - "id": 41, - "md5": "", - "size": 0, - "tags": "anime bare_midriff color drawing gentle giantess " - "karbo looking_at_tinies negeyari outdoors smiling " - "snake_girl white_hair", - "width": 1387, - }, - }), - ("https://giantessbooru.com/post/view/41"), - ("https://tentaclerape.net/post/view/10", { - "pattern": r"https://tentaclerape\.net/\./index\.php" - r"\?q=/image/10\.jpg", - "content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", - "keyword": { - "extension": "jpg", - "file_url": "https://tentaclerape.net/./index.php" - "?q=/image/10.jpg", - "filename": "10", - "height": 427, - "id": 10, - "md5": "945db71eeccaef82ce44b77564260c0b", - "size": 0, - "subcategory": "post", - "tags": "Deviant_Art Pet Tentacle artist_sche blonde_hair " - "blouse boots green_eyes highheels leash miniskirt " - "octopus schoolgirl white_skin willing", - "width": 300, - }, - }), - # video - ("https://tentaclerape.net/post/view/91267", { - "pattern": r"https://tentaclerape\.net/\./index\.php" - r"\?q=/image/91267\.mp4", - }), - ("https://booru.cavemanon.xyz/index.php?q=post/view/8335", { - "pattern": r"https://booru\.cavemanon\.xyz" - r"/index\.php\?q=image/8335\.png", - "content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", - "keyword": { - "extension": "png", - "file_url": "https://booru.cavemanon.xyz" - "/index.php?q=image/8335.png", - "filename": "8335", - "height": 460, - "id": 8335, - "md5": "", - "size": 0, - "tags": "Color Fang Food Pterodactyl " - "discord_emote transparent", - "width": 459, - }, - }), - ) + example = "https://loudbooru.com/post/view/12345" def __init__(self, match): Shimmie2Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/shopify.py b/gallery_dl/extractor/shopify.py index f6e8bc03..a658cac4 100644 --- a/gallery_dl/extractor/shopify.py +++ b/gallery_dl/extractor/shopify.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -95,24 +95,7 @@ class ShopifyCollectionExtractor(ShopifyExtractor): subcategory = "collection" directory_fmt = ("{category}", "{collection[title]}") pattern = BASE_PATTERN + r"(/collections/[\w-]+)/?(?:$|[?#])" - test = ( - ("https://chelseacrew.com/collections/flats"), - ("https://www.fashionnova.com/collections/mini-dresses", { - "range": "1-20", - "count": 20, - }), - ("https://www.fashionnova.com/collections/mini-dresses/?page=1"), - ("https://www.fashionnova.com/collections/mini-dresses#1"), - ("https://loungeunderwear.com/collections/apparel"), - ("https://michaels.com.au/collections/microphones"), - ("https://modcloth.com/collections/shoes"), - ("https://www.ohpolly.com/collections/dresses-mini-dresses"), - ("https://www.omgmiamiswimwear.com/collections/fajas"), - ("https://pinupgirlclothing.com/collections/evening"), - ("https://www.raidlondon.com/collections/flats"), - ("https://www.unique-vintage.com/collections/flapper-1920s"), - ("https://www.windsorstore.com/collections/dresses-ball-gowns"), - ) + example = "https://www.fashionnova.com/collections/TITLE" def metadata(self): return self.request(self.item_url + ".json").json() @@ -134,32 +117,7 @@ class ShopifyProductExtractor(ShopifyExtractor): subcategory = "product" directory_fmt = ("{category}", "Products") pattern = BASE_PATTERN + r"((?:/collections/[\w-]+)?/products/[\w-]+)" - test = ( - ("https://chelseacrew.com/collections/flats/products/dora"), - ("https://www.fashionnova.com/products/essential-slide-red", { - "pattern": r"https?://cdn\d*\.shopify.com/", - "count": 3, - }), - ("https://www.fashionnova.com/collections/flats/products/name"), - ("https://de.loungeunderwear.com/products/ribbed-crop-top-black"), - ("https://michaels.com.au/collections/audio/products" - "/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281"), - ("https://modcloth.com/collections/shoes/products/heidii-brn"), - (("https://www.ohpolly.com/products/edonia-ruched-triangle-cup" - "-a-line-mini-dress-brown")), - ("https://www.omgmiamiswimwear.com/products/la-medusa-maxi-dress", { - "pattern": r"https://cdn\.shopify\.com/s/files/1/1819/6171/", - "count": 5, - }), - ("https://pinupgirlclothing.com/collections/evening/products" - "/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design"), - ("https://www.raidlondon.com/collections/flats/products" - "/raid-addyson-chunky-flat-shoe-in-white"), - ("https://www.unique-vintage.com/collections/flapper-1920s/products" - "/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress"), - ("https://www.windsorstore.com/collections/accessories-belts/products" - "/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001"), - ) + example = "https://www.fashionnova.com/collections/TITLE/products/NAME" def products(self): product = self.request(self.item_url + ".json").json()["product"] diff --git a/gallery_dl/extractor/simplyhentai.py b/gallery_dl/extractor/simplyhentai.py index d1ccc492..6f722916 100644 --- a/gallery_dl/extractor/simplyhentai.py +++ b/gallery_dl/extractor/simplyhentai.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2021 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,20 +19,7 @@ class SimplyhentaiGalleryExtractor(GalleryExtractor): pattern = (r"(?:https?://)?(?!videos\.)([\w-]+\.)?simply-hentai\.com" r"(?!/(?:album|gifs?|images?|series)(?:/|$))" r"((?:/(?!(?:page|all-pages)(?:/|\.|$))[^/?#]+)+)") - test = ( - (("https://original-work.simply-hentai.com" - "/amazon-no-hiyaku-amazon-elixir"), { - "url": "21613585ae5ec2f69ea579e9713f536fceab5bd5", - "keyword": "9e87a0973553b2922ddee37958b8f5d87910af72", - }), - ("https://www.simply-hentai.com/notfound", { - "exception": exception.GalleryDLException, - }), - # custom subdomain - ("https://pokemon.simply-hentai.com/mao-friends-9bc39"), - # www subdomain, two path segments - ("https://www.simply-hentai.com/vocaloid/black-magnet"), - ) + example = "https://www.simply-hentai.com/TITLE" def __init__(self, match): subdomain, path = match.groups() @@ -89,17 +76,7 @@ class SimplyhentaiImageExtractor(Extractor): archive_fmt = "{token}" pattern = (r"(?:https?://)?(?:www\.)?(simply-hentai\.com" r"/(image|gif)/[^/?#]+)") - test = ( - (("https://www.simply-hentai.com/image" - "/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6"), { - "url": "3d8eb55240a960134891bd77fe1df7988fcdc455", - "keyword": "e10e5588481cab68329ef6ec1e5325206b2079a2", - }), - ("https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", { - "url": "f73916527211b4a40f26568ee26cd8999f5f4f30", - "keyword": "f94d775177fed918759c8a78a50976f867425b48", - }), - ) + example = "https://www.simply-hentai.com/image/NAME" def __init__(self, match): Extractor.__init__(self, match) @@ -142,20 +119,7 @@ class SimplyhentaiVideoExtractor(Extractor): filename_fmt = "{title}{episode:?_//>02}.{extension}" archive_fmt = "{title}_{episode}" pattern = r"(?:https?://)?(videos\.simply-hentai\.com/[^/?#]+)" - test = ( - ("https://videos.simply-hentai.com/creamy-pie-episode-02", { - "pattern": r"https://www\.googleapis\.com/drive/v3/files" - r"/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", - "keyword": "706790708b14773efc1e075ddd3b738a375348a5", - "options": (("verify", False),), - "count": 1, - }), - (("https://videos.simply-hentai.com" - "/1715-tifa-in-hentai-gang-bang-3d-movie"), { - "url": "ad9a36ae06c601b6490e3c401834b4949d947eb0", - "keyword": "f9dad94fbde9c95859e631ff4f07297a9567b874", - }), - ) + example = "https://videos.simply-hentai.com/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index b643c6f2..f10ab622 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -145,53 +145,7 @@ class SkebPostExtractor(SkebExtractor): """Extractor for a single skeb post""" subcategory = "post" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/works/(\d+)" - test = ("https://skeb.jp/@kanade_cocotte/works/38", { - "count": 2, - "keyword": { - "anonymous": False, - "body": "re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", - "client": { - "avatar_url": r"re:https://pbs.twimg.com/profile_images" - r"/\d+/\w+\.jpg", - "header_url": r"re:https://pbs.twimg.com/profile_banners" - r"/1375007870291300358/\d+/1500x500", - "id": 1196514, - "name": str, - "screen_name": "minato_ragi", - }, - "content_category": "preview", - "creator": { - "avatar_url": "https://pbs.twimg.com/profile_images" - "/1225470417063645184/P8_SiB0V.jpg", - "header_url": "https://pbs.twimg.com/profile_banners" - "/71243217/1647958329/1500x500", - "id": 159273, - "name": "イチノセ奏", - "screen_name": "kanade_cocotte", - }, - "file_id": int, - "file_url": str, - "genre": "art", - "nsfw": False, - "original": { - "byte_size": int, - "duration": None, - "extension": "re:psd|png", - "frame_rate": None, - "height": 3727, - "is_movie": False, - "width": 2810, - }, - "post_num": "38", - "post_url": "https://skeb.jp/@kanade_cocotte/works/38", - "source_body": None, - "source_thanks": None, - "tags": list, - "thanks": None, - "translated_body": False, - "translated_thanks": None, - } - }) + example = "https://skeb.jp/@USER/works/123" def __init__(self, match): SkebExtractor.__init__(self, match) @@ -205,13 +159,7 @@ class SkebUserExtractor(SkebExtractor): """Extractor for all posts from a skeb user""" subcategory = "user" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/?$" - test = ("https://skeb.jp/@kanade_cocotte", { - "pattern": r"https://skeb\.imgix\.net/uploads/origins/[\w-]+" - r"\?bg=%23fff&auto=format&txtfont=bold&txtshad=70" - r"&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150" - r"&txt=SAMPLE&fm=webp&w=800&s=\w+", - "range": "1-5", - }) + example = "https://skeb.jp/@USER" def posts(self): url = "{}/api/users/{}/works".format(self.root, self.user_name) @@ -230,10 +178,7 @@ class SkebSearchExtractor(SkebExtractor): """Extractor for skeb search results""" subcategory = "search" pattern = r"(?:https?://)?skeb\.jp/search\?q=([^&#]+)" - test = ("https://skeb.jp/search?q=bunny%20tree&t=works", { - "count": ">= 18", - "keyword": {"search_tags": "bunny tree"}, - }) + example = "https://skeb.jp/search?q=QUERY" def metadata(self): return {"search_tags": text.unquote(self.user_name)} @@ -286,7 +231,7 @@ class SkebFollowingExtractor(SkebExtractor): """Extractor for all creators followed by a skeb user""" subcategory = "following" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/following_creators" - test = ("https://skeb.jp/@user/following_creators",) + example = "https://skeb.jp/@USER/following_creators" def items(self): for user in self.users(): diff --git a/gallery_dl/extractor/slickpic.py b/gallery_dl/extractor/slickpic.py index 3727c0b0..b2961e17 100644 --- a/gallery_dl/extractor/slickpic.py +++ b/gallery_dl/extractor/slickpic.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -12,7 +12,6 @@ from .common import Extractor, Message from .. import text import time - BASE_PATTERN = r"(?:https?://)?([\w-]+)\.slickpic\.com" @@ -34,22 +33,7 @@ class SlickpicAlbumExtractor(SlickpicExtractor): filename_fmt = "{num:>03}_{id}{title:?_//}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/albums/([^/?#]+)" - test = ( - ("https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", { - "pattern": r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,," - r"/20160807/\w+/p/o/JSBFSS-\d+\.jpg", - "keyword": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", - "count": 102, - }), - ("https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", { - "range": "34", - "content": ("276eb2c902187bb177ae8013e310e1d6641fba9a", - "52b5a310587de1048030ab13a912f6a3a9cc7dab", - "cec6630e659dc72db1ee1a9a6f3b525189261988", - "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", - "22e83645fc242bc3584eca7ec982c8a53a4d8a44"), - }), - ) + example = "https://USER.slickpic.com/albums/TITLE/" def __init__(self, match): SlickpicExtractor.__init__(self, match) @@ -127,13 +111,7 @@ class SlickpicAlbumExtractor(SlickpicExtractor): class SlickpicUserExtractor(SlickpicExtractor): subcategory = "user" pattern = BASE_PATTERN + r"(?:/gallery)?/?(?:$|[?#])" - test = ( - ("https://mattcrandall.slickpic.com/gallery/", { - "count": ">= 358", - "pattern": SlickpicAlbumExtractor.pattern, - }), - ("https://mattcrandall.slickpic.com/"), - ) + example = "https://USER.slickpic.com/" def items(self): page = self.request(self.root + "/gallery?viewer").text diff --git a/gallery_dl/extractor/slideshare.py b/gallery_dl/extractor/slideshare.py index 3521298f..b56ed27b 100644 --- a/gallery_dl/extractor/slideshare.py +++ b/gallery_dl/extractor/slideshare.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2023 Mike Fährmann, Leonardo Taccari +# Copyright 2016-2017 Leonardo Taccari +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,45 +22,7 @@ class SlidesharePresentationExtractor(GalleryExtractor): archive_fmt = "{presentation}_{num}" pattern = (r"(?:https?://)?(?:www\.)?slideshare\.net" r"/(?:mobile/)?([^/?#]+)/([^/?#]+)") - test = ( - (("https://www.slideshare.net" - "/Slideshare/get-started-with-slide-share"), { - "pattern": r"https://image\.slidesharecdn\.com/getstartedwithslide" - r"share-150520173821-lva1-app6892/95/get-started-with-s" - r"lide-share-\d+-1024\.jpg\?cb=\d+", - "count": 19, - "content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", - "keyword": { - "description": "Get Started with SlideShare - " - "A Beginngers Guide for Creators", - "likes": int, - "presentation": "get-started-with-slide-share", - "date": "dt:2015-05-20 17:38:21", - "title": "Getting Started With SlideShare", - "user": "Slideshare", - "views": int, - }, - }), - # long title and description - (("https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren" - "-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem"), { - "url": "d8952260f8bec337dd809a958ec8091350393f6b", - "keyword": { - "title": "Warum Sie nicht Ihren Mitarbeitenden ändern " - "sollten, sondern Ihr Managementsystem", - "description": "Mitarbeitende verhalten sich mehrheitlich so, " - "wie das System es ihnen vorgibt. Welche Voraus" - "setzungen es braucht, damit Ihre Mitarbeitende" - "n ihr ganzes Herzblut einsetzen, bespricht Fre" - "di Schmidli in diesem Referat.", - }, - }), - # mobile URL - (("https://www.slideshare.net" - "/mobile/uqudent/introduction-to-fixed-prosthodontics"), { - "url": "72c431cb1eccbb6794f608ecbbc01d52e8768159", - }), - ) + example = "https://www.slideshare.net/USER/PRESENTATION" def __init__(self, match): self.user, self.presentation = match.groups() diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py index b9edd4ab..3639c0b5 100644 --- a/gallery_dl/extractor/smugmug.py +++ b/gallery_dl/extractor/smugmug.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2022 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -68,19 +68,7 @@ class SmugmugAlbumExtractor(SmugmugExtractor): directory_fmt = ("{category}", "{User[NickName]}", "{Album[Name]}") archive_fmt = "a_{Album[AlbumKey]}_{Image[ImageKey]}" pattern = r"smugmug:album:([^:]+)$" - test = ( - ("smugmug:album:cr4C7f", { - "url": "2c2e576e47d4e9ce60b44871f08a8c66b5ebaceb", - }), - # empty - ("smugmug:album:Fb7hMs", { - "count": 0, - }), - # no "User" - ("smugmug:album:6VRT8G", { - "url": "c4a0f4c4bfd514b93cbdeb02b3345bf7ef6604df", - }), - ) + example = "smugmug:album:ID" def __init__(self, match): SmugmugExtractor.__init__(self, match) @@ -107,18 +95,7 @@ class SmugmugImageExtractor(SmugmugExtractor): subcategory = "image" archive_fmt = "{Image[ImageKey]}" pattern = BASE_PATTERN + r"(?:/[^/?#]+)+/i-([^/?#-]+)" - test = ( - ("https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", { - "url": "e6408fd2c64e721fd146130dceb56a971ceb4259", - "keyword": "b31a63d07c9c26eb0f79f52d60d171a98938f99b", - "content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", - }), - # video - ("https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", { - "url": "04d0ab1ff829ca7d78f5acb5548953df08e9a5ee", - "keyword": "2b545184592c282b365fcbb7df6ca7952b8a3173", - }), - ) + example = "https://USER.smugmug.com/PATH/i-ID" def __init__(self, match): SmugmugExtractor.__init__(self, match) @@ -139,35 +116,13 @@ class SmugmugPathExtractor(SmugmugExtractor): """Extractor for smugmug albums from URL paths and users""" subcategory = "path" pattern = BASE_PATTERN + r"((?:/[^/?#a-fh-mo-z][^/?#]*)*)/?$" - test = ( - ("https://tdm.smugmug.com/Nature/Dove", { - "pattern": "smugmug:album:cr4C7f$", - }), - ("https://tdm.smugmug.com/", { - "pattern": SmugmugAlbumExtractor.pattern, - "url": "1640028712875b90974e5aecd91b60e6de6138c7", - }), - # gallery node without owner - ("https://www.smugmug.com/gallery/n-GLCjnD/", { - "pattern": "smugmug:album:6VRT8G$", - }), - # custom domain - ("smugmug:www.sitkapics.com/TREES-and-TRAILS/", { - "pattern": "smugmug:album:ct8Nds$", - }), - ("smugmug:www.sitkapics.com/", { - "pattern": r"smugmug:album:\w{6}$", - "count": ">= 14", - }), - ("smugmug:https://www.sitkapics.com/"), - ) + example = "https://USER.smugmug.com/PATH" def __init__(self, match): SmugmugExtractor.__init__(self, match) self.domain, self.user, self.path = match.groups() def items(self): - if not self.user: self.user = self.api.site_user(self.domain)["NickName"] diff --git a/gallery_dl/extractor/soundgasm.py b/gallery_dl/extractor/soundgasm.py index 236f94f6..7c75aaae 100644 --- a/gallery_dl/extractor/soundgasm.py +++ b/gallery_dl/extractor/soundgasm.py @@ -51,29 +51,7 @@ class SoundgasmAudioExtractor(SoundgasmExtractor): """Extractor for audio clips from soundgasm.net""" subcategory = "audio" pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)" - test = ( - (("https://soundgasm.net/u/ClassWarAndPuppies2" - "/687-Otto-von-Toontown-12822"), { - "pattern": r"https://media\.soundgasm\.net/sounds" - r"/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", - "keyword": { - "description": "We celebrate today’s important prisoner swap, " - "and finally bring the 2022 mid-terms to a clos" - "e with Raphael Warnock’s defeat of Herschel Wa" - "lker in Georgia. Then, we take a look at the Q" - "anon-addled attempt to overthrow the German go" - "vernment and install Heinrich XIII Prince of R" - "euss as kaiser.", - "extension": "m4a", - "filename": "26cb2b23b2f2c6094b40ee3a9167271e274b570a", - "slug": "687-Otto-von-Toontown-12822", - "title": "687 - Otto von Toontown (12/8/22)", - "user": "ClassWarAndPuppies2", - }, - }), - ("https://www.soundgasm.net/user/ClassWarAndPuppies2" - "/687-Otto-von-Toontown-12822"), - ) + example = "https://soundgasm.net/u/USER/TITLE" def __init__(self, match): SoundgasmExtractor.__init__(self, match) @@ -87,19 +65,7 @@ class SoundgasmUserExtractor(SoundgasmExtractor): """Extractor for all sounds from a soundgasm user""" subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)/?$" - test = ("https://soundgasm.net/u/fierce-aphrodite", { - "pattern": r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", - "count" : ">= 15", - "keyword": { - "description": str, - "extension": "m4a", - "filename": "re:^[0-9a-f]{40}$", - "slug": str, - "title": str, - "url": str, - "user": "fierce-aphrodite" - }, - }) + example = "https://soundgasm.net/u/USER" def __init__(self, match): SoundgasmExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/speakerdeck.py b/gallery_dl/extractor/speakerdeck.py index 9eb87617..e44fdae7 100644 --- a/gallery_dl/extractor/speakerdeck.py +++ b/gallery_dl/extractor/speakerdeck.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://speakerdeck.com/""" +"""Extractors for https://speakerdeck.com/""" from .common import Extractor, Message from .. import text @@ -19,16 +19,8 @@ class SpeakerdeckPresentationExtractor(Extractor): directory_fmt = ("{category}", "{user}") filename_fmt = "{presentation}-{num:>02}.{extension}" archive_fmt = "{presentation}_{num}" - pattern = (r"(?:https?://)?(?:www\.)?speakerdeck\.com" - r"/([^/?#]+)/([^/?#]+)") - test = ( - (("https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck"), { - "pattern": r"https://files.speakerdeck.com/presentations/" - r"50021f75cf1db900020005e7/slide_\d+.jpg", - "content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", - "count": 6, - }), - ) + pattern = r"(?:https?://)?(?:www\.)?speakerdeck\.com/([^/?#]+)/([^/?#]+)" + example = "https://speakerdeck.com/USER/PRESENTATION" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py index 32c17d76..6b4cba21 100644 --- a/gallery_dl/extractor/subscribestar.py +++ b/gallery_dl/extractor/subscribestar.py @@ -139,35 +139,7 @@ class SubscribestarUserExtractor(SubscribestarExtractor): """Extractor for media from a subscribestar user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!posts/)([^/?#]+)" - test = ( - ("https://www.subscribestar.com/subscribestar", { - "count": ">= 20", - "pattern": r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net" - r"/uploads(_v2)?/users/11/", - "keyword": { - "author_id": 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content": str, - "date" : "type:datetime", - "id" : int, - "num" : int, - "post_id": int, - "type" : "re:image|video|attachment", - "url" : str, - "?pinned": bool, - }, - }), - ("https://www.subscribestar.com/subscribestar", { - "options": (("metadata", True),), - "keyword": {"date": "type:datetime"}, - "range": "1", - }), - ("https://subscribestar.adult/kanashiipanda", { - "range": "1-10", - "count": 10, - }), - ) + example = "https://www.subscribestar.com/USER" def posts(self): needle_next_page = 'data-role="infinite_scroll-next_page" href="' @@ -189,32 +161,7 @@ class SubscribestarPostExtractor(SubscribestarExtractor): """Extractor for media from a single subscribestar post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.subscribestar.com/posts/102468", { - "count": 1, - "keyword": { - "author_id": 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content": "re:<h1>Brand Guidelines and Assets</h1>", - "date": "dt:2020-05-07 12:33:00", - "extension": "jpg", - "filename": "8ff61299-b249-47dc-880a-cdacc9081c62", - "group": "imgs_and_videos", - "height": 291, - "id": 203885, - "num": 1, - "pinned": False, - "post_id": 102468, - "type": "image", - "width": 700, - }, - }), - ("https://subscribestar.adult/posts/22950", { - "count": 1, - "keyword": {"date": "dt:2019-04-28 07:32:00"}, - }), - ) + example = "https://www.subscribestar.com/posts/12345" def posts(self): url = "{}/posts/{}".format(self.root, self.item) diff --git a/gallery_dl/extractor/szurubooru.py b/gallery_dl/extractor/szurubooru.py index 8c816ad1..5415bf30 100644 --- a/gallery_dl/extractor/szurubooru.py +++ b/gallery_dl/extractor/szurubooru.py @@ -95,15 +95,7 @@ class SzurubooruTagExtractor(SzurubooruExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}_{version}" pattern = BASE_PATTERN + r"/posts/query=([^/?#]+)" - test = ( - ("https://booru.foalcon.com/posts/query=simple_background", { - "pattern": r"https://booru\.foalcon\.com/data/posts" - r"/\d+_[0-9a-f]{16}\.\w+", - "range": "1-150", - "count": 150, - }), - ("https://booru.bcbnsfw.space/posts/query=simple_background"), - ) + example = "https://booru.foalcon.com/posts/query=TAG" def __init__(self, match): SzurubooruExtractor.__init__(self, match) @@ -121,19 +113,7 @@ class SzurubooruPostExtractor(SzurubooruExtractor): subcategory = "post" archive_fmt = "{id}_{version}" pattern = BASE_PATTERN + r"/post/(\d+)" - test = ( - ("https://booru.foalcon.com/post/30092", { - "pattern": r"https://booru\.foalcon\.com/data/posts" - r"/30092_b7d56e941888b624\.png", - "url": "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", - "content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", - }), - ("https://booru.bcbnsfw.space/post/1599", { - "pattern": r"https://booru\.bcbnsfw\.space/data/posts" - r"/1599_53784518e92086bd\.png", - "content": "0c38fc612ba1f03950fad31c4f80a1fccdab1096", - }), - ) + example = "https://booru.foalcon.com/post/12345" def __init__(self, match): SzurubooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/tapas.py b/gallery_dl/extractor/tapas.py index 53338e9a..bfca7a62 100644 --- a/gallery_dl/extractor/tapas.py +++ b/gallery_dl/extractor/tapas.py @@ -107,16 +107,7 @@ class TapasExtractor(Extractor): class TapasSeriesExtractor(TapasExtractor): subcategory = "series" pattern = BASE_PATTERN + r"/series/([^/?#]+)" - test = ( - ("https://tapas.io/series/just-leave-me-be", { - "pattern": r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", - "count": 132, - }), - # mature - ("https://tapas.io/series/yona", { - "count": 26, - }), - ) + example = "https://tapas.io/series/TITLE" def __init__(self, match): TapasExtractor.__init__(self, match) @@ -152,53 +143,7 @@ class TapasSeriesExtractor(TapasExtractor): class TapasEpisodeExtractor(TapasExtractor): subcategory = "episode" pattern = BASE_PATTERN + r"/episode/(\d+)" - test = ("https://tapas.io/episode/2068651", { - "url": "0b53644c864a0a097f65accea6bb620be9671078", - "pattern": "^text:", - "keyword": { - "book": True, - "comment_cnt": int, - "date": "dt:2021-02-23 16:02:07", - "early_access": False, - "escape_title": "You are a Tomb Raider (2)", - "free": True, - "id": 2068651, - "like_cnt": int, - "liked": bool, - "mature": False, - "next_ep_id": 2068652, - "nsfw": False, - "nu": False, - "num": 1, - "open_comments": True, - "pending_scene": 2, - "prev_ep_id": 2068650, - "publish_date": "2021-02-23T16:02:07Z", - "read": bool, - "related_ep_id": None, - "relative_publish_date": "Feb 23, 2021", - "scene": 2, - "scheduled": False, - "title": "You are a Tomb Raider (2)", - "unlock_cnt": 0, - "unlocked": False, - "view_cnt": int, - - "series": { - "genre": dict, - "has_book_cover": True, - "has_top_banner": True, - "id": 199931, - "premium": True, - "sale_type": "PAID", - "subscribed": bool, - "thumbsup_cnt": int, - "title": "Tomb Raider King", - "type": "BOOKS", - "url": "tomb-raider-king-novel", - }, - }, - }) + example = "https://tapas.io/episode/12345" def __init__(self, match): TapasExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/tcbscans.py b/gallery_dl/extractor/tcbscans.py index b5a730a4..a3ef26cb 100644 --- a/gallery_dl/extractor/tcbscans.py +++ b/gallery_dl/extractor/tcbscans.py @@ -16,35 +16,7 @@ class TcbscansChapterExtractor(ChapterExtractor): category = "tcbscans" root = "https://tcbscans.com" pattern = BASE_PATTERN + r"(/chapters/\d+/[^/?#]+)" - test = ( - (("https://tcbscans.com" - "/chapters/4708/chainsaw-man-chapter-108"), { - "pattern": (r"https://cdn\.[^/]+" - r"/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+"), - "count" : 17, - "keyword": { - "manga": "Chainsaw Man", - "chapter": 108, - "chapter_minor": "", - "lang": "en", - "language": "English", - }, - }), - ("https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", { - "pattern": (r"https://cdn\.[^/]+" - r"/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+"), - "count" : 18, - "keyword": { - "manga": "One Piece", - "chapter": 1065, - "chapter_minor": "", - "lang": "en", - "language": "English", - }, - }), - (("https://onepiecechapters.com/" - "chapters/44/ace-novel-manga-adaptation-chapter-1")), - ) + example = "https://tcbscans.com/chapters/12345/MANGA-chapter-123" def images(self, page): return [ @@ -70,19 +42,7 @@ class TcbscansMangaExtractor(MangaExtractor): root = "https://tcbscans.com" chapterclass = TcbscansChapterExtractor pattern = BASE_PATTERN + r"(/mangas/\d+/[^/?#]+)" - test = ( - ("https://tcbscans.com/mangas/13/chainsaw-man", { - "pattern": TcbscansChapterExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://onepiecechapters.com/mangas/4/jujutsu-kaisen", { - "pattern": TcbscansChapterExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://onepiecechapters.com/mangas/15/hunter-x-hunter"), - ) + example = "https://tcbscans.com/mangas/123/MANGA" def chapters(self, page): data = { diff --git a/gallery_dl/extractor/telegraph.py b/gallery_dl/extractor/telegraph.py index 116f3afa..dd5988f8 100644 --- a/gallery_dl/extractor/telegraph.py +++ b/gallery_dl/extractor/telegraph.py @@ -18,72 +18,7 @@ class TelegraphGalleryExtractor(GalleryExtractor): filename_fmt = "{num_formatted}_{filename}.{extension}" archive_fmt = "{slug}_{num}" pattern = r"(?:https?://)(?:www\.)??telegra\.ph(/[^/?#]+)" - test = ( - ("https://telegra.ph/Telegraph-Test-03-28", { - "pattern": r"https://telegra\.ph/file/[0-9a-f]+\.png", - "keyword": { - "author": "mikf", - "caption": r"re:test|", - "count": 2, - "date": "dt:2022-03-28 16:01:36", - "description": "Just a test", - "post_url": "https://telegra.ph/Telegraph-Test-03-28", - "slug": "Telegraph-Test-03-28", - "title": "Telegra.ph Test", - }, - }), - ("https://telegra.ph/森-03-28", { - "pattern": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - "count": 1, - "keyword": { - "author": "&", - "caption": "kokiri", - "count": 1, - "date": "dt:2022-03-28 16:31:26", - "description": "コキリの森", - "extension": "jpg", - "filename": "3ea79d23b0dd0889f215a", - "num": 1, - "num_formatted": "1", - "post_url": "https://telegra.ph/森-03-28", - "slug": "森-03-28", - "title": '"森"', - "url": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - }, - }), - ("https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", { - "url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", - "pattern": r"^https://pith1\.ru/uploads" - r"/posts/2019-12/\d+_\d+\.jpg$", - "keyword": { - "author": "Shotacon - заходи сюда", - "caption": "", - "count": 19, - "date": "dt:2022-05-27 16:17:27", - "description": "", - "num_formatted": r"re:^\d{2}$", - "post_url": "https://telegra.ph" - "/Vsyo-o-druzyah-moej-sestricy-05-27", - "slug": "Vsyo-o-druzyah-moej-sestricy-05-27", - "title": "Всё о друзьях моей сестрицы", - }, - }), - ("https://telegra.ph/Disharmonica---Saber-Nero-02-21", { - "pattern": r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", - "keyword": { - "author": "cosmos", - "caption": "", - "count": 89, - "date": "dt:2022-02-21 05:57:39", - "description": "", - "num_formatted": r"re:^\d{2}$", - "post_url": "https://telegra.ph" - "/Disharmonica---Saber-Nero-02-21", - "slug": "Disharmonica---Saber-Nero-02-21", - "title": "Disharmonica - Saber Nero", - }, - }), - ) + example = "https://telegra.ph/TITLE" def metadata(self, page): extr = text.extract_from(page) diff --git a/gallery_dl/extractor/test.py b/gallery_dl/extractor/test.py index 5d81ed51..e3f9f744 100644 --- a/gallery_dl/extractor/test.py +++ b/gallery_dl/extractor/test.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2017 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -33,11 +33,7 @@ class TestExtractor(Extractor): """ category = "test" pattern = r"t(?:est)?:([^:]*)(?::([^:]*)(?::(\*|[\d,]*))?)?$" - test = ( - ("test:pixiv"), - ("test:pixiv:user,favorite:0"), - ("test:"), - ) + example = "test:CATEGORY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index c6be38dd..78ff265d 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -99,28 +99,7 @@ class ToyhouseArtExtractor(ToyhouseExtractor): """Extractor for artworks of a toyhouse user""" subcategory = "art" pattern = BASE_PATTERN + r"/([^/?#]+)/art" - - test = ( - ("https://www.toyhou.se/d-floe/art", { - "range": "1-30", - "count": 30, - "pattern": r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se" - r"/images/\d+_\w+\.\w+$", - "keyword": { - "artists": list, - "characters": list, - "date": "type:datetime", - "hash": r"re:\w+", - "id": r"re:\d+", - "url": str, - "user": "d-floe", - }, - }), - # protected by Content Warning - ("https://www.toyhou.se/kroksoc/art", { - "count": ">= 19", - }), - ) + example = "https://www.toyhou.se/USER/art" def posts(self): return self._pagination("/{}/art".format(self.user)) @@ -136,37 +115,7 @@ class ToyhouseImageExtractor(ToyhouseExtractor): r"(?:www\.)?toyhou\.se/~images|" r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" r")/(\d+)") - test = ( - ("https://toyhou.se/~images/40587320", { - "content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", - "keyword": { - "artists": ["d-floe"], - "characters": ["Sumi"], - "date": "dt:2021-10-08 01:32:47", - "extension": "png", - "filename": "40587320_TT1NaBUr3FLkS1p", - "hash": "TT1NaBUr3FLkS1p", - "id": "40587320", - "url": "https://f2.toyhou.se/file/f2-toyhou-se/images" - "/40587320_TT1NaBUr3FLkS1p.png", - }, - }), - # direct link, multiple artists - (("https://f2.toyhou.se/file/f2-toyhou-se" - "/watermarks/36817425_bqhGcwcnU.png?1625561467"), { - "keyword": { - "artists": [ - "http://aminoapps.com/p/92sf3z", - "kroksoc (Color)"], - "characters": ["❀Reiichi❀"], - "date": "dt:2021-07-03 20:02:02", - "hash": "bqhGcwcnU", - "id": "36817425", - }, - }), - ("https://f2.toyhou.se/file/f2-toyhou-se" - "/images/40587320_TT1NaBUr3FLkS1p.png"), - ) + example = "https://toyhou.se/~images/12345" def posts(self): url = "{}/~images/{}".format(self.root, self.user) diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index e7d5226a..de7cdfc0 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -44,34 +44,7 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): """Extractor for image galleries on tsumino.com""" pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com" r"/(?:entry|Book/Info|Read/(?:Index|View))/(\d+)") - test = ( - ("https://www.tsumino.com/entry/40996", { - "pattern": r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", - "keyword": { - "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", - "gallery_id": 40996, - "date" : "dt:2018-06-29 00:00:00", - "count" : 42, - "collection": "", - "artist" : ["Itou Life"], - "group" : ["Itou Life"], - "parody" : list, - "characters": list, - "tags" : list, - "type" : "Doujinshi", - "rating" : float, - "uploader" : "sehki", - "lang" : "en", - "language" : "English", - "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", - }, - }), - ("https://www.tsumino.com/Book/Info/40996"), - ("https://www.tsumino.com/Read/View/45834"), - ("https://www.tsumino.com/Read/Index/45834"), - ) + example = "https://www.tsumino.com/entry/12345" def __init__(self, match): self.gallery_id = match.group(1) @@ -131,21 +104,8 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): class TsuminoSearchExtractor(TsuminoBase, Extractor): """Extractor for search results on tsumino.com""" subcategory = "search" - pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com" - r"/(?:Books/?)?#(.+)") - test = ( - ("https://www.tsumino.com/Books#?Character=Reimu+Hakurei", { - "pattern": TsuminoGalleryExtractor.pattern, - "range": "1-40", - "count": 40, - }), - (("http://www.tsumino.com/Books#~(Tags~(~" - "(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~" - "(Type~'1~Text~'Pantyhose~Exclude~false)))#"), { - "pattern": TsuminoGalleryExtractor.pattern, - "count": ">= 3", - }), - ) + pattern = r"(?i)(?:https?://)?(?:www\.)?tsumino\.com/(?:Books/?)?#(.+)" + example = "https://www.tsumino.com/Books#QUERY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index e8c99294..3dab16e8 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -273,64 +273,7 @@ class TumblrUserExtractor(TumblrExtractor): """Extractor for a Tumblr user's posts""" subcategory = "user" pattern = BASE_PATTERN + r"(?:/page/\d+|/archive)?/?$" - test = ( - ("http://demo.tumblr.com/", { - "pattern": r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_\d+\.jpg", - "count": 1, - "options": (("posts", "photo"),), - }), - ("http://demo.tumblr.com/", { - "pattern": (r"https?://(?:$|" - r"\d+\.media\.tumblr\.com/.+_1280\.jpg|" - r"a\.tumblr\.com/tumblr_\w+)"), - "count": 3, - "options": (("posts", "all"), ("external", True)) - }), - # dashboard-only - ("https://mikf123-hidden.tumblr.com/", { - "options": (("access-token", None),), - "exception": exception.AuthorizationError, - }), - # dashboard-only - ("https://mikf123-hidden.tumblr.com/", { - "count": 2, - "keyword": {"tags": ["test", "hidden"]}, - }), - # password protected - ("https://mikf123-private.tumblr.com/", { - "count": 2, - "keyword": {"tags": ["test", "private"]}, - }), - # dashboard-only & password protected - ("https://mikf123-private-hidden.tumblr.com/", { - "count": 2, - "keyword": {"tags": ["test", "private", "hidden"]}, - }), - # date-min/-max/-format (#337) - ("https://mikf123.tumblr.com/", { - "count": 4, - "options": (("date-min", "201804"), ("date-max", "201805"), - ("date-format", "%Y%m")) - }), - # pagination with 'date-max' (#2191) and 'api-key' - ("https://donttrustthetits.tumblr.com/", { - "options": ( - ("access-token", None), - ("original", False), - ("date-max", "2015-04-25T00:00:00"), - ("date-min", "2015-04-01T00:00:00"), - ), - "count": 316, - }), - ("https://demo.tumblr.com/page/2"), - ("https://demo.tumblr.com/archive"), - ("tumblr:http://www.b-authentique.com/"), - ("tumblr:www.b-authentique.com"), - ("https://www.tumblr.com/blog/view/smarties-art"), - ("https://www.tumblr.com/blog/smarties-art"), - ("https://www.tumblr.com/smarties-art"), - ) + example = "https://www.tumblr.com/BLOG" def posts(self): return self.api.posts(self.blog, {}) @@ -340,58 +283,7 @@ class TumblrPostExtractor(TumblrExtractor): """Extractor for a single Tumblr post""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:post/|image/)?(\d+)" - test = ( - ("http://demo.tumblr.com/post/459265350", { - "pattern": (r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_1280.jpg"), - "count": 1, - }), - ("https://mikf123.tumblr.com/post/167770226574/text-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/181022561719/quote-post", { - "count": 1, - }), - ("https://mikf123.tumblr.com/post/167623351559/link-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/167633596145/video-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/167770026604/audio-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/172687798174/photo-post", { - "count": 4, - }), - ("https://mikf123.tumblr.com/post/181022380064/chat-post", { - "count": 0, - }), - # high-quality images (#1846) - ("https://kichatundk.tumblr.com/post/654953419288821760", { - "count": 2, - "content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", - }), - # high-quality images (#1344) - ("https://hameru-is-cool.tumblr.com/post/639261855227002880", { - "count": 2, - "content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", - }), - ("https://mikf123.tumblr.com/image/689860196535762944", { - "pattern": r"^https://\d+\.media\.tumblr\.com" - r"/134791621559a79793563b636b5fe2c6" - r"/8f1131551cef6e74-bc/s99999x99999" - r"/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", - }), - # HTML response (#297) - ("http://ziemniax.tumblr.com/post/109697912859/", { - "exception": exception.NotFoundError, - }), - ("http://demo.tumblr.com/image/459265350"), - ("https://www.tumblr.com/blog/view/smarties-art/686047436641353728"), - ("https://www.tumblr.com/blog/smarties-art/686047436641353728"), - ("https://www.tumblr.com/smarties-art/686047436641353728"), - ) + example = "https://www.tumblr.com/BLOG/12345" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -411,16 +303,7 @@ class TumblrTagExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by tag""" subcategory = "tag" pattern = BASE_PATTERN + r"/tagged/([^/?#]+)" - test = ( - ("http://demo.tumblr.com/tagged/Times%20Square", { - "pattern": r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_1280.jpg", - "count": 1, - }), - ("https://www.tumblr.com/blog/view/smarties-art/tagged/undertale"), - ("https://www.tumblr.com/blog/smarties-art/tagged/undertale"), - ("https://www.tumblr.com/smarties-art/tagged/undertale"), - ) + example = "https://www.tumblr.com/BLOG/tagged/TAG" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -434,18 +317,7 @@ class TumblrDayExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by day""" subcategory = "day" pattern = BASE_PATTERN + r"/day/(\d\d\d\d/\d\d/\d\d)" - test = ( - ("https://mikf123.tumblr.com/day/2018/01/05", { - "pattern": r"https://64\.media\.tumblr\.com" - r"/1a2be8c63f1df58abd2622861696c72a" - r"/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", - "keyword": {"id": 169341068404}, - "count": 1, - }), - ("https://www.tumblr.com/blog/view/mikf123/day/2018/01/05"), - ("https://www.tumblr.com/blog/mikf123/day/2018/01/05"), - ("https://www.tumblr.com/mikf123/day/2018/01/05"), - ) + example = "https://www.tumblr.com/BLOG/day/1970/01/01" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -468,14 +340,7 @@ class TumblrLikesExtractor(TumblrExtractor): directory_fmt = ("{category}", "{blog_name}", "likes") archive_fmt = "f_{blog[name]}_{id}_{num}" pattern = BASE_PATTERN + r"/likes" - test = ( - ("http://mikf123.tumblr.com/likes", { - "count": 1, - }), - ("https://www.tumblr.com/blog/view/mikf123/likes"), - ("https://www.tumblr.com/blog/mikf123/likes"), - ("https://www.tumblr.com/mikf123/likes"), - ) + example = "https://www.tumblr.com/BLOG/likes" def posts(self): return self.api.likes(self.blog) diff --git a/gallery_dl/extractor/tumblrgallery.py b/gallery_dl/extractor/tumblrgallery.py index 6940f3e6..27cc9d0c 100644 --- a/gallery_dl/extractor/tumblrgallery.py +++ b/gallery_dl/extractor/tumblrgallery.py @@ -38,7 +38,7 @@ class TumblrgalleryTumblrblogExtractor(TumblrgalleryExtractor): """Extractor for Tumblrblog on tumblrgallery.xyz""" subcategory = "tumblrblog" pattern = BASE_PATTERN + r"(/tumblrblog/gallery/(\d+)\.html)" - test = ("https://tumblrgallery.xyz/tumblrblog/gallery/103975.html",) + example = "https://tumblrgallery.xyz/tumblrblog/gallery/12345.html" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) @@ -69,11 +69,7 @@ class TumblrgalleryPostExtractor(TumblrgalleryExtractor): """Extractor for Posts on tumblrgallery.xyz""" subcategory = "post" pattern = BASE_PATTERN + r"(/post/(\d+)\.html)" - test = ("https://tumblrgallery.xyz/post/405674.html", { - "pattern": r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711d" - r"ec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", - "count": 3, - }) + example = "https://tumblrgallery.xyz/post/12345.html" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) @@ -98,10 +94,7 @@ class TumblrgallerySearchExtractor(TumblrgalleryExtractor): filename_fmt = "{category}_{num:>03}_{gallery_id}_{id}_{title}.{extension}" directory_fmt = ("{category}", "{search_term}") pattern = BASE_PATTERN + r"(/s\.php\?q=([^&#]+))" - test = ("https://tumblrgallery.xyz/s.php?q=everyday-life", { - "pattern": r"https://\d+\.media\.tumblr\.com/.+", - "count": "< 1000", - }) + example = "https://tumblrgallery.xyz/s.php?q=QUERY" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index c3e0a262..49c84195 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -19,12 +19,12 @@ class TwibooruExtractor(BooruExtractor): """Base class for twibooru extractors""" category = "twibooru" basecategory = "philomena" + root = "https://twibooru.org" filename_fmt = "{id}_{filename}.{extension}" archive_fmt = "{id}" request_interval = 6.05 page_start = 1 per_page = 50 - root = "https://twibooru.org" def _init(self): self.api = TwibooruAPI(self) @@ -46,50 +46,7 @@ class TwibooruPostExtractor(TwibooruExtractor): subcategory = "post" request_interval = 1.0 pattern = BASE_PATTERN + r"/(\d+)" - test = ("https://twibooru.org/1", { - "pattern": r"https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", - "keyword": { - "animated": False, - "aspect_ratio": 1.0, - "comment_count": int, - "created_at": "2020-07-08T22:26:55.743Z", - "date": "dt:2020-07-08 22:26:55", - "description": "Why have I done this?", - "downvotes": 0, - "duration": 0.0, - "faves": int, - "first_seen_at": "2020-07-08T22:26:55.743Z", - "format": "png", - "height": 576, - "hidden_from_users": False, - "id": 1, - "intensities": dict, - "locations": [], - "media_type": "image", - "mime_type": "image/png", - "name": "1676547__safe_artist-colon-scraggleman_oc_oc-colon-" - "floor+bored_oc+only_bags+under+eyes_bust_earth+pony_" - "female_goggles_helmet_mare_meme_neet_neet+home+g.png", - "orig_sha512_hash": "re:8b4c00d2[0-9a-f]{120}", - "processed": True, - "representations": dict, - "score": int, - "sha512_hash": "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf6" - "34fbb181b32f44070aa588938e26c4eb072b1eb61489aa" - "f3062fb644a76c79f936b97723a2c3e0e5d3", - "size": 70910, - "source_url": "", - "tag_ids": list, - "tags": list, - "thumbnails_generated": True, - "updated_at": "2022-11-27T00:34:50.483Z", - "upvotes": int, - "view_url": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "width": 576, - "wilson_score": float, - }, - }) + example = "https://twibooru.org/12345" def __init__(self, match): TwibooruExtractor.__init__(self, match) @@ -104,16 +61,7 @@ class TwibooruSearchExtractor(TwibooruExtractor): subcategory = "search" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?:search/?\?([^#]+)|tags/([^/?#]+))" - test = ( - ("https://twibooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://twibooru.org/tags/cute", { - "range": "1-20", - "count": 20, - }), - ) + example = "https://twibooru.org/search?q=TAG" def __init__(self, match): TwibooruExtractor.__init__(self, match) @@ -147,19 +95,7 @@ class TwibooruGalleryExtractor(TwibooruExtractor): directory_fmt = ("{category}", "galleries", "{gallery[id]} {gallery[title]}") pattern = BASE_PATTERN + r"/galleries/(\d+)" - test = ("https://twibooru.org/galleries/1", { - "range": "1-20", - "keyword": { - "gallery": { - "description": "Best nation pone and " - "russian related pics.", - "id": 1, - "spoiler_warning": "Russia", - "thumbnail_id": 694923, - "title": "Marussiaverse", - }, - }, - }) + example = "https://twibooru.org/galleries/12345" def __init__(self, match): TwibooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index bf925f15..d8890099 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -480,22 +480,7 @@ class TwitterUserExtractor(TwitterExtractor): subcategory = "user" pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" r"|i(?:/user/|ntent/user\?user_id=)(\d+))") - test = ( - ("https://twitter.com/supernaturepics", { - "options": (("include", "all"),), - "pattern": r"https://twitter\.com/supernaturepics" - r"/(photo|header_photo|timeline|tweets" - r"|media|with_replies|likes)$", - "count": 7, - }), - ("https://mobile.twitter.com/supernaturepics?p=i"), - ("https://www.twitter.com/id:2976459548"), - ("https://twitter.com/i/user/2976459548"), - ("https://twitter.com/intent/user?user_id=2976459548"), - ("https://fxtwitter.com/supernaturepics"), - ("https://vxtwitter.com/supernaturepics"), - ("https://x.com/supernaturepics"), - ) + example = "https://twitter.com/USER" def __init__(self, match): TwitterExtractor.__init__(self, match) @@ -523,22 +508,7 @@ class TwitterTimelineExtractor(TwitterExtractor): """Extractor for a Twitter user timeline""" subcategory = "timeline" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/timeline(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/timeline", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - # suspended account (#2216) - ("https://twitter.com/OptionalTypo/timeline", { - "exception": exception.NotFoundError, - }), - # suspended account user ID - ("https://twitter.com/id:772949683521978368/timeline", { - "exception": exception.NotFoundError, - }), - ("https://mobile.twitter.com/supernaturepics/timeline#t"), - ("https://www.twitter.com/id:2976459548/timeline"), - ) + example = "https://twitter.com/USER/timeline" def tweets(self): # yield initial batch of (media) tweets @@ -583,14 +553,7 @@ class TwitterTweetsExtractor(TwitterExtractor): """Extractor for Tweets from a user's Tweets timeline""" subcategory = "tweets" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/tweets(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/tweets", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/tweets#t"), - ("https://www.twitter.com/id:2976459548/tweets"), - ) + example = "https://twitter.com/USER/tweets" def tweets(self): return self.api.user_tweets(self.user) @@ -600,14 +563,7 @@ class TwitterRepliesExtractor(TwitterExtractor): """Extractor for Tweets from a user's timeline including replies""" subcategory = "replies" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/with_replies", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/with_replies#t"), - ("https://www.twitter.com/id:2976459548/with_replies"), - ) + example = "https://twitter.com/USER/with_replies" def tweets(self): return self.api.user_tweets_and_replies(self.user) @@ -617,14 +573,7 @@ class TwitterMediaExtractor(TwitterExtractor): """Extractor for Tweets from a user's Media timeline""" subcategory = "media" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/media", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/media#t"), - ("https://www.twitter.com/id:2976459548/media"), - ) + example = "https://twitter.com/USER/media" def tweets(self): return self.api.user_media(self.user) @@ -634,7 +583,7 @@ class TwitterLikesExtractor(TwitterExtractor): """Extractor for liked tweets""" subcategory = "likes" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/likes(?!\w)" - test = ("https://twitter.com/supernaturepics/likes",) + example = "https://twitter.com/USER/likes" def metadata(self): return {"user_likes": self.user} @@ -653,7 +602,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): """Extractor for bookmarked tweets""" subcategory = "bookmark" pattern = BASE_PATTERN + r"/i/bookmarks()" - test = ("https://twitter.com/i/bookmarks",) + example = "https://twitter.com/i/bookmarks" def tweets(self): return self.api.user_bookmarks() @@ -669,11 +618,7 @@ class TwitterListExtractor(TwitterExtractor): """Extractor for Twitter lists""" subcategory = "list" pattern = BASE_PATTERN + r"/i/lists/(\d+)/?$" - test = ("https://twitter.com/i/lists/784214683683127296", { - "range": "1-40", - "count": 40, - "archive": False, - }) + example = "https://twitter.com/i/lists/12345" def tweets(self): return self.api.list_latest_tweets_timeline(self.user) @@ -683,11 +628,7 @@ class TwitterListMembersExtractor(TwitterExtractor): """Extractor for members of a Twitter list""" subcategory = "list-members" pattern = BASE_PATTERN + r"/i/lists/(\d+)/members" - test = ("https://twitter.com/i/lists/784214683683127296/members", { - "pattern": TwitterTimelineExtractor.pattern, - "range": "1-40", - "count": 40, - }) + example = "https://twitter.com/i/lists/12345/members" def items(self): self.login() @@ -698,10 +639,7 @@ class TwitterFollowingExtractor(TwitterExtractor): """Extractor for followed users""" subcategory = "following" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/following"), - ("https://www.twitter.com/id:2976459548/following"), - ) + example = "https://twitter.com/USER/following" def items(self): self.login() @@ -712,11 +650,7 @@ class TwitterSearchExtractor(TwitterExtractor): """Extractor for Twitter search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" - test = ("https://twitter.com/search?q=nature", { - "range": "1-20", - "count": 20, - "archive": False, - }) + example = "https://twitter.com/search?q=QUERY" def metadata(self): return {"search": text.unquote(self.user)} @@ -747,10 +681,7 @@ class TwitterHashtagExtractor(TwitterExtractor): """Extractor for Twitter hashtags""" subcategory = "hashtag" pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" - test = ("https://twitter.com/hashtag/nature", { - "pattern": TwitterSearchExtractor.pattern, - "url": "3571c3a53b7647ea35517041fdc17f77ec5b2cb9", - }) + example = "https://twitter.com/hashtag/NAME" def items(self): url = "{}/search?q=%23{}".format(self.root, self.user) @@ -764,10 +695,7 @@ class TwitterEventExtractor(TwitterExtractor): directory_fmt = ("{category}", "Events", "{event[id]} {event[short_title]}") pattern = BASE_PATTERN + r"/i/events/(\d+)" - test = ("https://twitter.com/i/events/1484669206993903616", { - "range": "1-20", - "count": ">=1", - }) + example = "https://twitter.com/i/events/12345" def metadata(self): return {"event": self.api.live_event(self.user)} @@ -780,188 +708,7 @@ class TwitterTweetExtractor(TwitterExtractor): """Extractor for images from individual tweets""" subcategory = "tweet" pattern = BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" - test = ( - ("https://twitter.com/supernaturepics/status/604341487988576256", { - "url": "88a40f7d25529c2501c46f2218f9e0de9aa634b4", - "content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", - }), - # 4 images - ("https://twitter.com/perrypumas/status/894001459754180609", { - "url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", - }), - # video - ("https://twitter.com/perrypumas/status/1065692031626829824", { - "pattern": r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", - }), - # content with emoji, newlines, hashtags (#338) - ("https://twitter.com/playpokemon/status/1263832915173048321", { - "keyword": {"content": ( - r"re:Gear up for #PokemonSwordShieldEX with special Mystery " - "Gifts! \n\nYou’ll be able to receive four Galarian form " - "Pokémon with Hidden Abilities, plus some very useful items. " - "It’s our \\(Mystery\\) Gift to you, Trainers! \n\n❓🎁➡️ " - )}, - }), - # Reply to deleted tweet (#403, #838) - ("https://twitter.com/i/web/status/1170041925560258560", { - "pattern": r"https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", - }), - # 'replies' option (#705) - ("https://twitter.com/i/web/status/1170041925560258560", { - "options": (("replies", False),), - "count": 0, - }), - # 'replies' to self (#1254) - ("https://twitter.com/i/web/status/1424882930803908612", { - "options": (("replies", "self"),), - "count": 4, - "keyword": {"user": { - "description": "re:business email-- rhettaro.bloom@gmail.com " - "patreon- http://patreon.com/Princecanary", - "url": "http://princecanary.tumblr.com", - }}, - }), - ("https://twitter.com/i/web/status/1424898916156284928", { - "options": (("replies", "self"),), - "count": 1, - }), - # "quoted" option (#854) - ("https://twitter.com/StobiesGalaxy/status/1270755918330896395", { - "options": (("quoted", True),), - "pattern": r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", - "count": 8, - }), - # quoted tweet (#526, #854) - ("https://twitter.com/StobiesGalaxy/status/1270755918330896395", { - "pattern": r"https://pbs\.twimg\.com/media/EaK.+=jpg", - "count": 4, - }), - # different 'user' and 'author' in quoted Tweet (#3922) - ("https://twitter.com/web/status/1644907989109751810", { - "keyword": { - "author": {"id": 321629993 , "name": "Cakes_Comics"}, - "user" : {"id": 718928225360080897, "name": "StobiesGalaxy"}, - }, - }), - # TwitPic embeds (#579) - ("https://twitter.com/i/web/status/112900228289540096", { - "options": (("twitpic", True), ("cards", False)), - "pattern": r"https://\w+.cloudfront.net/photos/large/\d+.jpg", - "count": 2, # 1 duplicate - }), - # TwitPic URL not in 'urls' (#3792) - ("https://twitter.com/shimoigusaP/status/8138669971", { - "options": (("twitpic", True),), - "pattern": r"https://\w+.cloudfront.net/photos/large/\d+.png", - "count": 1, - }), - # Twitter card (#1005) - ("https://twitter.com/billboard/status/1306599586602135555", { - "options": (("cards", True),), - "pattern": r"https://pbs.twimg.com/card_img/\d+/", - }), - # unified_card image_website (#2875) - ("https://twitter.com/i/web/status/1561674543323910144", { - "options": (("cards", True),), - "pattern": r"https://pbs\.twimg\.com/media/F.+=jpg", - }), - # unified_card image_carousel_website - ("https://twitter.com/doax_vv_staff/status/1479438945662685184", { - "options": (("cards", True),), - "pattern": r"https://pbs\.twimg\.com/media/F.+=png", - "count": 6, - }), - # unified_card video_website (#2875) - ("https://twitter.com/bang_dream_1242/status/1561548715348746241", { - "options": (("cards", True),), - "pattern": r"https://video\.twimg\.com/amplify_video" - r"/1560607284333449216/vid/720x720/\w+\.mp4", - }), - # unified_card without type - ("https://twitter.com/i/web/status/1466183847628865544", { - "count": 0, - }), - # 'cards-blacklist' option - ("https://twitter.com/i/web/status/1571141912295243776", { - "options": (("cards", "ytdl"), - ("cards-blacklist", ("twitch.tv",))), - "count": 0, - }), - # retweet - ("https://twitter.com/jessica_3978/status/1296304589591810048", { - "options": (("retweets", True),), - "count": 2, - "keyword": { - "tweet_id" : 1296304589591810048, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:34:32", - "date_original": "dt:2020-08-20 04:00:28", - }, - }), - # original retweets (#1026) - ("https://twitter.com/jessica_3978/status/1296304589591810048", { - "options": (("retweets", "original"),), - "count": 2, - "keyword": { - "tweet_id" : 1296296016002547713, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:00:28", - "date_original": "dt:2020-08-20 04:00:28", - }, - }), - # all Tweets from a 'conversation' (#1319) - ("https://twitter.com/supernaturepics/status/604341487988576256", { - "options": (("conversations", True),), - "count": 5, - }), - # retweet with missing media entities (#1555) - ("https://twitter.com/morino_ya/status/1392763691599237121", { - "options": (("retweets", True),), - "count": 0, # private - }), - # deleted quote tweet (#2225) - ("https://twitter.com/i/web/status/1460044411165888515", { - "count": 0, - }), - # "Misleading" content - ("https://twitter.com/i/web/status/1486373748911575046", { - "count": 4, - }), - # age-restricted (#2354) - ("https://twitter.com/mightbecursed/status/1492954264909479936", { - "options": (("syndication", True),), - "keyword": {"date": "dt:2022-02-13 20:10:09"}, - "count": 1, - }), - # media alt texts / descriptions (#2617) - ("https://twitter.com/my0nruri/status/1528379296041299968", { - "keyword": {"description": "oc"} - }), - # '?format=...&name=...'-style URLs - ("https://twitter.com/poco_dandy/status/1150646424461176832", { - "options": (("cards", True),), - "pattern": r"https://pbs.twimg.com/card_img/157\d+/[\w-]+" - r"\?format=(jpg|png)&name=orig$", - "range": "1-2", - }), - # note tweet with long 'content' - ("https://twitter.com/i/web/status/1629193457112686592", { - "keyword": { - "content": """\ -BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have \ -just contradicted federal government regulators, saying that toxic air \ -pollutants in East Palestine, Ohio, could pose long-term risks. \n\nThe \ -Washington Post writes, "Three weeks after the toxic train derailment in \ -Ohio, an analysis of Environmental Protection Agency data has found nine air \ -pollutants at levels that could raise long-term health concerns in and around \ -East Palestine, according to an independent analysis. \n\n\"The analysis by \ -Texas A&M University seems to contradict statements by state and federal \ -regulators that air near the crash site is completely safe, despite residents \ -complaining about rashes, breathing problems and other health effects." \ -Your reaction.""", - }, - }), - ) + example = "https://twitter.com/USER/status/12345" def __init__(self, match): TwitterExtractor.__init__(self, match) @@ -1042,21 +789,7 @@ class TwitterAvatarExtractor(TwitterExtractor): filename_fmt = "avatar {date}.{extension}" archive_fmt = "AV_{user[id]}_{date}" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/photo" - test = ( - ("https://twitter.com/supernaturepics/photo", { - "pattern": r"https://pbs\.twimg\.com/profile_images" - r"/554585280938659841/FLVAlX18\.jpeg", - "keyword": { - "date": "dt:2015-01-12 10:26:49", - "extension": "jpeg", - "filename": "FLVAlX18", - "tweet_id": 554585280938659841, - }, - }), - ("https://twitter.com/User16/photo", { - "count": 0, - }), - ) + example = "https://twitter.com/USER/photo" def tweets(self): self.api._user_id_by_screen_name(self.user) @@ -1078,20 +811,7 @@ class TwitterBackgroundExtractor(TwitterExtractor): filename_fmt = "background {date}.{extension}" archive_fmt = "BG_{user[id]}_{date}" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/header_photo" - test = ( - ("https://twitter.com/supernaturepics/header_photo", { - "pattern": r"https://pbs\.twimg\.com/profile_banners" - r"/2976459548/1421058583", - "keyword": { - "date": "dt:2015-01-12 10:29:43", - "filename": "1421058583", - "tweet_id": 554586009367478272, - }, - }), - ("https://twitter.com/User16/header_photo", { - "count": 0, - }), - ) + example = "https://twitter.com/USER/header_photo" def tweets(self): self.api._user_id_by_screen_name(self.user) @@ -1111,13 +831,7 @@ class TwitterImageExtractor(Extractor): category = "twitter" subcategory = "image" pattern = r"https?://pbs\.twimg\.com/media/([\w-]+)(?:\?format=|\.)(\w+)" - test = ( - ("https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", { - "options": (("size", "4096x4096,orig"),), - "url": "cb3042a6f6826923da98f0d2b66c427e9385114c", - }), - ("https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig"), - ) + example = "https://pbs.twimg.com/media/ABCDE?format=jpg&name=orig" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index b298c27e..a1b87b9c 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -76,66 +76,7 @@ class UnsplashImageExtractor(UnsplashExtractor): """Extractor for a single unsplash photo""" subcategory = "image" pattern = BASE_PATTERN + r"/photos/([^/?#]+)" - test = ("https://unsplash.com/photos/lsoogGC_5dg", { - "pattern": r"https://images\.unsplash\.com/photo-1586348943529-" - r"beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", - "keyword": { - "alt_description": "re:silhouette of trees near body of water ", - "blur_hash": "LZP4uQS4jboe%#o0WCa}2doJNaaz", - "? categories": list, - "color": "#f3c08c", - "created_at": "2020-04-08T12:29:42Z", - "date": "dt:2020-04-08 12:29:42", - "description": "The Island", - "downloads": int, - "exif": { - "aperture": "11", - "exposure_time": "30", - "focal_length": "70.0", - "iso": 200, - "make": "Canon", - "model": "Canon EOS 5D Mark IV" - }, - "extension": "jpg", - "filename": "photo-1586348943529-beaae6c28db9", - "height": 6272, - "id": "lsoogGC_5dg", - "liked_by_user": False, - "likes": int, - "location": { - "city": "Beaver Dam", - "country": "United States", - "name": "Beaver Dam, WI 53916, USA", - "position": { - "latitude": 43.457769, - "longitude": -88.837329, - }, - }, - "promoted_at": "2020-04-08T15:12:03Z", - "sponsorship": None, - "tags": list, - "updated_at": str, - "user": { - "accepted_tos": True, - "bio": str, - "first_name": "Dave", - "id": "uMJXuywXLiU", - "instagram_username": "just_midwest_rock", - "last_name": "Hoefler", - "location": None, - "name": "Dave Hoefler", - "portfolio_url": None, - "total_collections": int, - "total_likes": int, - "total_photos": int, - "twitter_username": None, - "updated_at": str, - "username": "davehoefler", - }, - "views": int, - "width": 4480, - }, - }) + example = "https://unsplash.com/photos/ID" def photos(self): url = "{}/napi/photos/{}".format(self.root, self.item) @@ -146,12 +87,7 @@ class UnsplashUserExtractor(UnsplashExtractor): """Extractor for all photos of an unsplash user""" subcategory = "user" pattern = BASE_PATTERN + r"/@(\w+)/?$" - test = ("https://unsplash.com/@davehoefler", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/@USER" def photos(self): url = "{}/napi/users/{}/photos".format(self.root, self.item) @@ -163,12 +99,7 @@ class UnsplashFavoriteExtractor(UnsplashExtractor): """Extractor for all likes of an unsplash user""" subcategory = "favorite" pattern = BASE_PATTERN + r"/@(\w+)/likes" - test = ("https://unsplash.com/@davehoefler/likes", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/@USER/likes" def photos(self): url = "{}/napi/users/{}/likes".format(self.root, self.item) @@ -180,18 +111,7 @@ class UnsplashCollectionExtractor(UnsplashExtractor): """Extractor for an unsplash collection""" subcategory = "collection" pattern = BASE_PATTERN + r"/collections/([^/?#]+)(?:/([^/?#]+))?" - test = ( - ("https://unsplash.com/collections/3178572/winter", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "keyword": {"collection_id": "3178572", - "collection_title": "winter"}, - "range": "1-30", - "count": 30, - }), - ("https://unsplash.com/collections/3178572/"), - ("https://unsplash.com/collections/_8qJQ2bCMWE/2021.05"), - ) + example = "https://unsplash.com/collections/12345/TITLE" def __init__(self, match): UnsplashExtractor.__init__(self, match) @@ -210,13 +130,7 @@ class UnsplashSearchExtractor(UnsplashExtractor): """Extractor for unsplash search results""" subcategory = "search" pattern = BASE_PATTERN + r"/s/photos/([^/?#]+)(?:\?([^#]+))?" - test = ("https://unsplash.com/s/photos/hair-style", { - "pattern": r"https://(images|plus)\.unsplash\.com" - r"/((flagged/|premium_)?photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/s/photos/QUERY" def __init__(self, match): UnsplashExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/uploadir.py b/gallery_dl/extractor/uploadir.py index bd18c0ac..ce34e7de 100644 --- a/gallery_dl/extractor/uploadir.py +++ b/gallery_dl/extractor/uploadir.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,40 +20,7 @@ class UploadirFileExtractor(Extractor): filename_fmt = "{filename} ({id}).{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?uploadir\.com/(?:user/)?u(?:ploads)?/([^/?#]+)" - test = ( - # image - ("https://uploadir.com/u/rd3t46ry", { - "pattern": r"https://uploadir\.com/u/rd3t46ry", - "count": 1, - "keyword": { - "extension": "jpg", - "filename": "Chloe and Rachel 4K jpg", - "id": "rd3t46ry", - }, - }), - # archive - ("https://uploadir.com/uploads/gxe8ti9v/downloads/new", { - "pattern": r"https://uploadir\.com/uploads/gxe8ti9v/downloads", - "count": 1, - "keyword": { - "extension": "zip", - "filename": "NYAN-Mods-Pack#1", - "id": "gxe8ti9v", - }, - }), - # utf-8 filename - ("https://uploadir.com/u/fllda6xl", { - "pattern": r"https://uploadir\.com/u/fllda6xl", - "count": 1, - "keyword": { - "extension": "png", - "filename": "_圖片_🖼_image_", - "id": "fllda6xl", - }, - }), - ("https://uploadir.com/uploads/rd3t46ry"), - ("https://uploadir.com/user/uploads/rd3t46ry"), - ) + example = "https://uploadir.com/u/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/urlshortener.py b/gallery_dl/extractor/urlshortener.py index 4b49a638..f2e65214 100644 --- a/gallery_dl/extractor/urlshortener.py +++ b/gallery_dl/extractor/urlshortener.py @@ -35,20 +35,7 @@ class UrlshortenerLinkExtractor(UrlshortenerExtractor): """Extractor for general-purpose URL shorteners""" subcategory = "link" pattern = BASE_PATTERN + r"/([^/?#]+)" - test = ( - ("https://bit.ly/3cWIUgq", { - "count": 1, - "pattern": "^https://gumroad.com/l/storm_b1", - }), - ("https://t.co/bCgBY8Iv5n", { - "count": 1, - "pattern": "^https://twitter.com/elonmusk/status/" - "1421395561324896257/photo/1", - }), - ("https://t.co/abcdefghij", { - "exception": exception.NotFoundError, - }), - ) + example = "https://bit.ly/abcde" def __init__(self, match): UrlshortenerExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index 6b1178e8..1ce969f7 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,10 +30,7 @@ class VanillarockPostExtractor(VanillarockExtractor): archive_fmt = "{filename}" pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?!category/|tag/)[^/?#]+)/?$") - test = ("https://vanilla-rock.com/mizuhashi_parsee-5", { - "url": "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", - "keyword": "b91df99b714e1958d9636748b1c81a07c3ef52c9", - }) + example = "https://vanilla-rock.com/TITLE" def items(self): extr = text.extract_from(self.request(self.root + self.path).text) @@ -66,18 +63,7 @@ class VanillarockTagExtractor(VanillarockExtractor): subcategory = "tag" pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?:tag|category)/[^?#]+)") - test = ( - ("https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", { - "pattern": VanillarockPostExtractor.pattern, - "count": ">= 12", - }), - (("https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83" - "%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95" - "%e3%83%88%e3%82%a8%e3%83%ad"), { - "pattern": VanillarockPostExtractor.pattern, - "count": ">= 5", - }), - ) + example = "https://vanilla-rock.com/tag/TAG" def items(self): url = self.root + self.path diff --git a/gallery_dl/extractor/vichan.py b/gallery_dl/extractor/vichan.py index 2fafb56e..79d7916e 100644 --- a/gallery_dl/extractor/vichan.py +++ b/gallery_dl/extractor/vichan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -40,33 +40,7 @@ class VichanThreadExtractor(VichanExtractor): filename_fmt = "{time}{num:?-//} {filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://8kun.top/test/res/65248.html", { - "pattern": r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", - "count": ">= 8", - }), - # old-style file URLs (#1101) - # ("https://8kun.top/d/res/13258.html", { - # "pattern": r"https://media\.128ducks\.com/d/src/\d+(-\d)?\.\w+", - # "range": "1-20", - # }), - - ("https://wikieat.club/cel/res/25321.html", { - "pattern": r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+", - "count": ">= 200", - }), - - ("https://smuglo.li/a/res/1154380.html", { - "pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+", - "count": ">= 18", - "keyword": { - "board": "a", - "thread": "1154380", - "title": "Mob Psycho 100 Season 3", - }, - }), - ("https://smugloli.net/a/res/1145409.html"), - ) + example = "https://8kun.top/a/res/12345.html" def __init__(self, match): VichanExtractor.__init__(self, match) @@ -123,28 +97,7 @@ class VichanBoardExtractor(VichanExtractor): """Extractor for vichan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" - test = ( - ("https://8kun.top/v/index.html", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://8kun.top/v/2.html"), - ("https://8kun.top/v/index.html?PageSpeed=noscript"), - - ("https://wikieat.club/cel/index.html", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://wikieat.club/cel/catalog.html"), - ("https://wikieat.club/cel/2.html"), - - ("https://smuglo.li/a", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://smuglo.li/a/1.html"), - ("https://smugloli.net/cute/catalog.html"), - ) + example = "https://8kun.top/a/" def __init__(self, match): VichanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vipergirls.py b/gallery_dl/extractor/vipergirls.py index 084f9b25..456a0298 100644 --- a/gallery_dl/extractor/vipergirls.py +++ b/gallery_dl/extractor/vipergirls.py @@ -72,17 +72,7 @@ class VipergirlsThreadExtractor(VipergirlsExtractor): """Extractor for vipergirls threads""" subcategory = "thread" pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?$" - test = ( - (("https://vipergirls.to/threads/4328304" - "-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000"), { - "url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", - "count": 225, - }), - ("https://vipergirls.to/threads/6858916-Karina/page4", { - "count": 1279, - }), - ("https://vipergirls.to/threads/4328304"), - ) + example = "https://vipergirls.to/threads/12345-TITLE" def __init__(self, match): VipergirlsExtractor.__init__(self, match) @@ -103,21 +93,7 @@ class VipergirlsPostExtractor(VipergirlsExtractor): subcategory = "post" pattern = (BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)") - test = ( - (("https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-" - "Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081"), { - "pattern": r"https://vipr\.im/\w{12}$", - "range": "2-113", - "count": 112, - "keyword": { - "id": "116038081", - "imagecount": "113", - "number": "116038081", - "thread_id": "4328304", - "title": "FemJoy Danica - Simply Beautiful (x112) 3000x4500", - }, - }), - ) + example = "https://vipergirls.to/threads/12345-TITLE?p=23456#post23456" def __init__(self, match): VipergirlsExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 56924521..c9cd02f3 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -105,43 +105,7 @@ class VkPhotosExtractor(VkExtractor): pattern = (BASE_PATTERN + r"/(?:" r"(?:albums|photos|id)(-?\d+)" r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))") - test = ( - ("https://vk.com/id398982326", { - "pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1" - r"/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", - "count": ">= 35", - "keyword": { - "id": r"re:\d+", - "user": { - "id": "398982326", - "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", - "name": "", - "nick": "Dobrov Kurva", - }, - }, - }), - ("https://vk.com/cosplayinrussia", { - "range": "15-25", - "keyword": { - "id": r"re:\d+", - "user": { - "id" : "-165740836", - "info": str, - "name": "cosplayinrussia", - "nick": "Косплей | Cosplay 18+", - }, - }, - }), - # photos without width/height (#2535) - ("https://vk.com/id76957806", { - "pattern": r"https://sun\d+-\d+\.userapi\.com/", - "range": "1-9", - "count": 9, - }), - ("https://m.vk.com/albums398982326"), - ("https://www.vk.com/id398982326?profile=1"), - ("https://vk.com/albums-165740836"), - ) + example = "https://vk.com/id12345" def __init__(self, match): VkExtractor.__init__(self, match) @@ -181,18 +145,7 @@ class VkAlbumExtractor(VkExtractor): subcategory = "album" directory_fmt = ("{category}", "{user[id]}", "{album[id]}") pattern = BASE_PATTERN + r"/album(-?\d+)_(\d+)$" - test = ( - ("https://vk.com/album-165740836_281339889", { - "count": 12, - }), - # "Access denied" (#2556) - ("https://vk.com/album-53775183_00", { - "exception": exception.AuthorizationError, - }), - ("https://vk.com/album232175027_00", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://vk.com/album12345_00" def __init__(self, match): VkExtractor.__init__(self, match) @@ -214,11 +167,7 @@ class VkTaggedExtractor(VkExtractor): subcategory = "tagged" directory_fmt = ("{category}", "{user[id]}", "tags") pattern = BASE_PATTERN + r"/tag(-?\d+)$" - test = ( - ("https://vk.com/tag304303884", { - "count": 44, - }), - ) + example = "https://vk.com/tag12345" def __init__(self, match): VkExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index 053a799f..14e3c7b4 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -116,16 +116,7 @@ class VscoUserExtractor(VscoExtractor): """Extractor for images from a user on vsco.co""" subcategory = "user" pattern = BASE_PATTERN + r"(?:/gallery|/images(?:/\d+)?)?/?(?:$|[?#])" - test = ( - ("https://vsco.co/missuri/gallery", { - "pattern": r"https://image(-aws.+)?\.vsco\.co" - r"/[0-9a-f/]+/[\w-]+\.\w+", - "range": "1-80", - "count": 80, - }), - ("https://vsco.co/missuri/images/1"), - ("https://vsco.co/missuri"), - ) + example = "https://vsco.co/USER/gallery" def images(self): url = "{}/{}/gallery".format(self.root, self.user) @@ -149,11 +140,7 @@ class VscoCollectionExtractor(VscoExtractor): directory_fmt = ("{category}", "{user}", "collection") archive_fmt = "c_{user}_{id}" pattern = BASE_PATTERN + r"/collection/" - test = ("https://vsco.co/vsco/collection/1", { - "pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", - "range": "1-80", - "count": 80, - }) + example = "https://vsco.co/USER/collection/12345" def images(self): url = "{}/{}/collection/1".format(self.root, self.user) @@ -176,29 +163,7 @@ class VscoImageExtractor(VscoExtractor): """Extractor for individual images on vsco.co""" subcategory = "image" pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)" - test = ( - ("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", { - "url": "a45f9712325b42742324b330c348b72477996031", - "content": "1394d070828d82078035f19a92f404557b56b83f", - "keyword": { - "id" : "5d34b93ef632433030707ce2", - "user" : "erenyildiz", - "grid" : "erenyildiz", - "meta" : dict, - "tags" : list, - "date" : "dt:2019-07-21 19:12:11", - "video" : False, - "width" : 1537, - "height": 1537, - "description": "re:Ni seviyorum. #vsco #vscox #vscochallenges", - }, - }), - ("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", { - "url": "08e7eef3301756ce81206c0b47c1e9373756a74a", - "content": "e739f058d726ee42c51c180a505747972a7dfa47", - "keyword": {"video" : True}, - }), - ) + example = "https://vsco.co/USER/media/0123456789abcdef" def __init__(self, match): VscoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index 9e271098..479e8a85 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -57,15 +57,7 @@ class WallhavenSearchExtractor(WallhavenExtractor): directory_fmt = ("{category}", "{search[q]}") archive_fmt = "s_{search[q]}_{id}" pattern = r"(?:https?://)?wallhaven\.cc/search(?:/?\?([^#]+))?" - test = ( - ("https://wallhaven.cc/search?q=touhou"), - (("https://wallhaven.cc/search?q=id%3A87" - "&categories=111&purity=100&sorting=date_added&order=asc&page=3"), { - "pattern": (r"https://w\.wallhaven\.cc" - r"/full/\w\w/wallhaven-\w+\.\w+"), - "count": "<= 30", - }), - ) + example = "https://wallhaven.cc/search?q=QUERY" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -83,9 +75,7 @@ class WallhavenCollectionExtractor(WallhavenExtractor): subcategory = "collection" directory_fmt = ("{category}", "{username}", "{collection_id}") pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/favorites/(\d+)" - test = ("https://wallhaven.cc/user/AksumkA/favorites/74", { - "count": ">= 50", - }) + example = "https://wallhaven.cc/user/USER/favorites/12345" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -102,7 +92,7 @@ class WallhavenUserExtractor(WallhavenExtractor): """Extractor for a wallhaven user""" subcategory = "user" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/?$" - test = ("https://wallhaven.cc/user/AksumkA/",) + example = "https://wallhaven.cc/user/USER" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -123,10 +113,7 @@ class WallhavenCollectionsExtractor(WallhavenExtractor): """Extractor for all collections of a wallhaven user""" subcategory = "collections" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/favorites/?$" - test = ("https://wallhaven.cc/user/AksumkA/favorites", { - "pattern": WallhavenCollectionExtractor.pattern, - "count": 4, - }) + example = "https://wallhaven.cc/user/USER/favorites" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -146,12 +133,7 @@ class WallhavenUploadsExtractor(WallhavenExtractor): directory_fmt = ("{category}", "{username}") archive_fmt = "u_{username}_{id}" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/uploads" - test = ("https://wallhaven.cc/user/AksumkA/uploads", { - "pattern": (r"https://[^.]+\.wallhaven\.cc" - r"/full/\w\w/wallhaven-\w+\.\w+"), - "range": "1-100", - "count": 100, - }) + example = "https://wallhaven.cc/user/USER/uploads" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -170,41 +152,7 @@ class WallhavenImageExtractor(WallhavenExtractor): subcategory = "image" pattern = (r"(?:https?://)?(?:wallhaven\.cc/w/|whvn\.cc/" r"|w\.wallhaven\.cc/[a-z]+/\w\w/wallhaven-)(\w+)") - test = ( - ("https://wallhaven.cc/w/01w334", { - "pattern": (r"https://[^.]+\.wallhaven\.cc" - r"/full/01/wallhaven-01w334\.jpg"), - "content": "497212679383a465da1e35bd75873240435085a2", - "keyword": { - "id" : "01w334", - "width" : 1920, - "height" : 1200, - "resolution" : "1920x1200", - "ratio" : "1.6", - "colors" : list, - "tags" : list, - "file_size" : 278799, - "file_type" : "image/jpeg", - "purity" : "sfw", - "short_url" : "https://whvn.cc/01w334", - "source" : str, - "uploader" : { - "group" : "Owner/Developer", - "username" : "AksumkA", - }, - "date" : "dt:2014-08-31 06:17:19", - "wh_category": "anime", - "views" : int, - "favorites" : int, - }, - }), - # NSFW - ("https://wallhaven.cc/w/dge6v3", { - "url": "e4b802e70483f659d790ad5d0bd316245badf2ec", - }), - ("https://whvn.cc/01w334"), - ("https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg"), - ) + example = "https://wallhaven.cc/w/ID" def __init__(self, match): WallhavenExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wallpapercave.py b/gallery_dl/extractor/wallpapercave.py index 6c3af767..bce1026d 100644 --- a/gallery_dl/extractor/wallpapercave.py +++ b/gallery_dl/extractor/wallpapercave.py @@ -18,9 +18,7 @@ class WallpapercaveImageExtractor(Extractor): subcategory = "image" root = "https://wallpapercave.com" pattern = r"(?:https?://)?(?:www\.)?wallpapercave\.com" - test = ("https://wallpapercave.com/w/wp10270355", { - "content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", - }) + example = "https://wallpapercave.com/w/wp12345" def items(self): page = self.request(text.ensure_http_scheme(self.url)).text diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index bdedfcbc..6f152ed7 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,17 +21,7 @@ class WarosuThreadExtractor(Extractor): filename_fmt = "{tim}-{filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" pattern = r"(?:https?://)?(?:www\.)?warosu\.org/([^/]+)/thread/(\d+)" - test = ( - ("https://warosu.org/jp/thread/16656025", { - "url": "889d57246ed67e491e5b8f7f124e50ea7991e770", - "keyword": "c00ea4c5460c5986994f17bb8416826d42ca57c0", - }), - ("https://warosu.org/jp/thread/16658073", { - "url": "4500cf3184b067424fd9883249bd543c905fbecd", - "keyword": "7534edf4ec51891dbf44d775b73fbbefd52eec71", - "content": "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", - }), - ) + example = "https://warosu.org/a/thread/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index c4d242a1..ddbfaa06 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -72,32 +72,7 @@ class WeasylExtractor(Extractor): class WeasylSubmissionExtractor(WeasylExtractor): subcategory = "submission" pattern = BASE_PATTERN + r"(?:~[\w~-]+/submissions|submission)/(\d+)" - test = ( - ("https://www.weasyl.com/~fiz/submissions/2031/a-wesley", { - "pattern": "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c29" - "40be928532785dfbf35c37622664d2fbb8114c3b063df969562fc5" - "1/fiz-a-wesley.png", - "keyword": { - "comments" : int, - "date" : "dt:2012-04-20 00:38:04", - "description" : "<p>(flex)</p>\n", - "favorites" : int, - "folder_name" : "Wesley Stuff", - "folderid" : 2081, - "friends_only": False, - "owner" : "Fiz", - "owner_login" : "fiz", - "rating" : "general", - "submitid" : 2031, - "subtype" : "visual", - "tags" : list, - "title" : "A Wesley!", - "type" : "submission", - "views" : int, - }, - }), - ("https://www.weasyl.com/submission/2031/a-wesley"), - ) + example = "https://www.weasyl.com/~USER/submissions/12345/TITLE" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -113,13 +88,7 @@ class WeasylSubmissionExtractor(WeasylExtractor): class WeasylSubmissionsExtractor(WeasylExtractor): subcategory = "submissions" pattern = BASE_PATTERN + r"(?:~|submissions/)([\w~-]+)/?$" - test = ( - ("https://www.weasyl.com/~tanidareal", { - "count": ">= 200" - }), - ("https://www.weasyl.com/submissions/tanidareal"), - ("https://www.weasyl.com/~aro~so") - ) + example = "https://www.weasyl.com/submissions/USER" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -134,9 +103,7 @@ class WeasylFolderExtractor(WeasylExtractor): subcategory = "folder" directory_fmt = ("{category}", "{owner_login}", "{folder_name}") pattern = BASE_PATTERN + r"submissions/([\w~-]+)\?folderid=(\d+)" - test = ("https://www.weasyl.com/submissions/tanidareal?folderid=7403", { - "count": ">= 12" - }) + example = "https://www.weasyl.com/submissions/USER?folderid=12345" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -157,14 +124,7 @@ class WeasylJournalExtractor(WeasylExtractor): filename_fmt = "{journalid} {title}.{extension}" archive_fmt = "{journalid}" pattern = BASE_PATTERN + r"journal/(\d+)" - test = ("https://www.weasyl.com/journal/17647/bbcode", { - "keyword": { - "title" : "BBCode", - "date" : "dt:2013-09-19 23:11:23", - "content": "<p><a>javascript:alert(42);</a></p>\n\n" - "<p>No more of that!</p>\n", - }, - }) + example = "https://www.weasyl.com/journal/12345" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -181,9 +141,7 @@ class WeasylJournalsExtractor(WeasylExtractor): filename_fmt = "{journalid} {title}.{extension}" archive_fmt = "{journalid}" pattern = BASE_PATTERN + r"journals/([\w~-]+)" - test = ("https://www.weasyl.com/journals/charmander", { - "count": ">= 2", - }) + example = "https://www.weasyl.com/journals/USER" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -203,9 +161,7 @@ class WeasylFavoriteExtractor(WeasylExtractor): subcategory = "favorite" directory_fmt = ("{category}", "{owner_login}", "Favorites") pattern = BASE_PATTERN + r"favorites\?userid=(\d+)" - test = ("https://www.weasyl.com/favorites?userid=184616&feature=submit", { - "count": ">= 5", - }) + example = "https://www.weasyl.com/favorites?userid=12345" def __init__(self, match): WeasylExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index b0384259..a09b7d04 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,36 +21,7 @@ class WebmshareVideoExtractor(Extractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?(?:s\d+\.)?webmshare\.com" r"/(?:play/|download-webm/)?(\w{3,})") - test = ( - ("https://webmshare.com/O9mWY", { - "keyword": { - "date": "dt:2022-12-04 00:00:00", - "extension": "webm", - "filename": "O9mWY", - "height": 568, - "id": "O9mWY", - "thumb": "https://s1.webmshare.com/t/O9mWY.jpg", - "title": "Yeah buddy over here", - "url": "https://s1.webmshare.com/O9mWY.webm", - "views": int, - "width": 320, - }, - }), - ("https://s1.webmshare.com/zBGAg.webm", { - "keyword": { - "date": "dt:2018-12-07 00:00:00", - "height": 1080, - "id": "zBGAg", - "thumb": "https://s1.webmshare.com/t/zBGAg.jpg", - "title": "", - "url": "https://s1.webmshare.com/zBGAg.webm", - "views": int, - "width": 1920, - }, - }), - ("https://webmshare.com/play/zBGAg"), - ("https://webmshare.com/download-webm/zBGAg"), - ) + example = "https://webmshare.com/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/webtoons.py b/gallery_dl/extractor/webtoons.py index 31592420..dc9a4f19 100644 --- a/gallery_dl/extractor/webtoons.py +++ b/gallery_dl/extractor/webtoons.py @@ -46,6 +46,8 @@ class WebtoonsEpisodeExtractor(WebtoonsBase, GalleryExtractor): archive_fmt = "{title_no}_{episode_no}_{num}" pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" r"/viewer(?:\?([^#'\"]+))") + example = ("https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" + "?title_no=123&episode_no=12345") test = ( (("https://www.webtoons.com/en/comedy/safely-endangered" "/ep-572-earth/viewer?title_no=352&episode_no=572"), { @@ -119,28 +121,7 @@ class WebtoonsComicExtractor(WebtoonsBase, Extractor): categorytransfer = True pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" r"/list(?:\?([^#]+))") - test = ( - # english - (("https://www.webtoons.com/en/comedy/live-with-yourself/" - "list?title_no=919"), { - "pattern": WebtoonsEpisodeExtractor.pattern, - "range": "1-15", - "count": ">= 15", - }), - # french - (("https://www.webtoons.com/fr/romance/subzero/" - "list?title_no=1845&page=3"), { - "count": ">= 15", - }), - # (#820) - (("https://www.webtoons.com/en/challenge/scoob-and-shag/" - "list?title_no=210827&page=9"), { - "count": ">= 18", - }), - # (#1643) - ("https://www.webtoons.com/es/romance/lore-olympus/" - "list?title_no=1725"), - ) + example = "https://www.webtoons.com/en/GENRE/TITLE/list?title_no=123" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index ae0fc4e3..51999455 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -219,16 +219,7 @@ class WeiboUserExtractor(WeiboExtractor): """Extractor for weibo user profiles""" subcategory = "user" pattern = USER_PATTERN + r"(?:$|#)" - test = ( - ("https://weibo.com/1758989602", { - "pattern": r"^https://weibo\.com/u/1758989602\?tabtype=feed$", - }), - ("https://weibo.com/u/1758989602"), - ("https://weibo.com/p/1758989602"), - ("https://m.weibo.cn/profile/2314621010"), - ("https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO"), - ("https://www.weibo.com/p/1003062314621010/home"), - ) + example = "https://weibo.com/USER" def initialize(self): pass @@ -248,10 +239,7 @@ class WeiboHomeExtractor(WeiboExtractor): """Extractor for weibo 'home' listings""" subcategory = "home" pattern = USER_PATTERN + r"\?tabtype=home" - test = ("https://weibo.com/1758989602?tabtype=home", { - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=home" def statuses(self): endpoint = "/profile/myhot" @@ -263,24 +251,7 @@ class WeiboFeedExtractor(WeiboExtractor): """Extractor for weibo user feeds""" subcategory = "feed" pattern = USER_PATTERN + r"\?tabtype=feed" - test = ( - ("https://weibo.com/1758989602?tabtype=feed", { - "range": "1-30", - "count": 30, - }), - ("https://weibo.com/zhouyuxi77?tabtype=feed", { - "keyword": {"status": {"user": {"id": 7488709788}}}, - "range": "1", - }), - ("https://www.weibo.com/n/周于希Sally?tabtype=feed", { - "keyword": {"status": {"user": {"id": 7488709788}}}, - "range": "1", - }), - # deleted (#2521) - ("https://weibo.com/u/7500315942?tabtype=feed", { - "count": 0, - }), - ) + example = "https://weibo.com/USER?tabtype=feed" def statuses(self): endpoint = "/statuses/mymblog" @@ -292,12 +263,7 @@ class WeiboVideosExtractor(WeiboExtractor): """Extractor for weibo 'video' listings""" subcategory = "videos" pattern = USER_PATTERN + r"\?tabtype=video" - test = ("https://weibo.com/1758989602?tabtype=video", { - "pattern": r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)" - r"/(../)?\w+\.mp4\?label=mp", - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=video" def statuses(self): endpoint = "/profile/getprofilevideolist" @@ -311,11 +277,7 @@ class WeiboNewvideoExtractor(WeiboExtractor): """Extractor for weibo 'newVideo' listings""" subcategory = "newvideo" pattern = USER_PATTERN + r"\?tabtype=newVideo" - test = ("https://weibo.com/1758989602?tabtype=newVideo", { - "pattern": r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=newVideo" def statuses(self): endpoint = "/profile/getWaterFallContent" @@ -327,9 +289,7 @@ class WeiboArticleExtractor(WeiboExtractor): """Extractor for weibo 'article' listings""" subcategory = "article" pattern = USER_PATTERN + r"\?tabtype=article" - test = ("https://weibo.com/1758989602?tabtype=article", { - "count": 0, - }) + example = "https://weibo.com/USER?tabtype=article" def statuses(self): endpoint = "/statuses/mymblog" @@ -341,12 +301,7 @@ class WeiboAlbumExtractor(WeiboExtractor): """Extractor for weibo 'album' listings""" subcategory = "album" pattern = USER_PATTERN + r"\?tabtype=album" - test = ("https://weibo.com/1758989602?tabtype=album", { - "pattern": r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)" - r"|g\.us\.sinaimg\.cn/../\w+\.mp4)", - "range": "1-3", - "count": 3, - }) + example = "https://weibo.com/USER?tabtype=album" def statuses(self): endpoint = "/profile/getImageWall" @@ -368,57 +323,7 @@ class WeiboStatusExtractor(WeiboExtractor): """Extractor for images from a status on weibo.cn""" subcategory = "status" pattern = BASE_PATTERN + r"/(detail|status|\d+)/(\w+)" - test = ( - ("https://m.weibo.cn/detail/4323047042991618", { - "pattern": r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", - "keyword": {"status": { - "count": 1, - "date": "dt:2018-12-30 13:56:36", - }}, - }), - ("https://m.weibo.cn/detail/4339748116375525", { - "pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", - }), - # unavailable video (#427) - ("https://m.weibo.cn/status/4268682979207023", { - "exception": exception.NotFoundError, - }), - # non-numeric status ID (#664) - ("https://weibo.com/3314883543/Iy7fj4qVg"), - # original retweets (#1542) - ("https://m.weibo.cn/detail/4600272267522211", { - "options": (("retweets", "original"),), - "keyword": {"status": {"id": 4600167083287033}}, - }), - # type == livephoto (#2146) - ("https://weibo.com/5643044717/KkuDZ4jAA", { - "range": "2,4,6", - "pattern": r"https://video\.weibo\.com/media/play\?livephoto=" - r"https%3A%2F%2Fus.sinaimg.cn%2F\w+\.mov", - }), - # type == gif - ("https://weibo.com/1758989602/LvBhm5DiP", { - "pattern": r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104" - r"120005tc0E010\.mp4\?label=gif_mp4", - }), - # missing 'playback_list' (#2792) - ("https://weibo.com/2909128931/4409545658754086", { - "count": 10, - }), - # empty 'playback_list' (#3301) - ("https://weibo.com/1501933722/4142890299009993", { - "pattern": r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01" - r"000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori" - r"=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", - "count": 1, - }), - # mix_media_info (#3793) - ("https://weibo.com/2427303621/MxojLlLgQ", { - "count": 9, - }), - ("https://m.weibo.cn/status/4339748116375525"), - ("https://m.weibo.cn/5746766133/4339748116375525"), - ) + example = "https://weibo.com/detail/12345" def statuses(self): status = self._status_by_id(self.user) diff --git a/gallery_dl/extractor/wikiart.py b/gallery_dl/extractor/wikiart.py index 0e068588..938c0485 100644 --- a/gallery_dl/extractor/wikiart.py +++ b/gallery_dl/extractor/wikiart.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -69,10 +69,7 @@ class WikiartArtistExtractor(WikiartExtractor): subcategory = "artist" directory_fmt = ("{category}", "{artist[artistName]}") pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)/?$" - test = ("https://www.wikiart.org/en/thomas-cole", { - "url": "6844f207a5063c499fc1d5651b03127bc4fe2f73", - "keyword": "09230b5f504697119e267349bf92487e657a7384", - }) + example = "https://www.wikiart.org/en/ARTIST" def __init__(self, match): WikiartExtractor.__init__(self, match) @@ -94,16 +91,7 @@ class WikiartImageExtractor(WikiartArtistExtractor): """Extractor for individual paintings on wikiart.org""" subcategory = "image" pattern = BASE_PATTERN + r"/(?!(?:paintings|artists)-by-)([\w-]+)/([\w-]+)" - test = ( - ("https://www.wikiart.org/en/thomas-cole/the-departure-1838", { - "url": "976cc2545f308a650b5dbb35c29d3cee0f4673b3", - "keyword": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", - }), - # no year or '-' in slug - ("https://www.wikiart.org/en/huang-shen/summer", { - "url": "d7f60118c34067b2b37d9577e412dc1477b94207", - }), - ) + example = "https://www.wikiart.org/en/ARTIST/TITLE" def __init__(self, match): WikiartArtistExtractor.__init__(self, match) @@ -125,9 +113,7 @@ class WikiartArtworksExtractor(WikiartExtractor): subcategory = "artworks" directory_fmt = ("{category}", "Artworks by {group!c}", "{type}") pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)" - test = ("https://www.wikiart.org/en/paintings-by-media/grisaille", { - "url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", - }) + example = "https://www.wikiart.org/en/paintings-by-GROUP/TYPE" def __init__(self, match): WikiartExtractor.__init__(self, match) @@ -147,10 +133,7 @@ class WikiartArtistsExtractor(WikiartExtractor): """Extractor for artist collections on wikiart.org""" subcategory = "artists" pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)") - test = ("https://www.wikiart.org/en/artists-by-century/12", { - "pattern": WikiartArtistExtractor.pattern, - "count": ">= 8", - }) + example = "https://www.wikiart.org/en/artists-by-GROUP/TYPE" def __init__(self, match): WikiartExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 5f02e94a..d3586c01 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -18,62 +18,7 @@ class WikifeetGalleryExtractor(GalleryExtractor): archive_fmt = "{type}_{celeb}_{pid}" pattern = (r"(?:https?://)(?:(?:www\.)?wikifeetx?|" r"men\.wikifeet)\.com/([^/?#]+)") - test = ( - ("https://www.wikifeet.com/Madison_Beer", { - "pattern": (r"https://pics\.wikifeet\.com/Madison_Beer" - r"-Feet-\d+\.jpg"), - "count" : ">= 352", - "keyword": { - "celeb" : "Madison_Beer", - "celebrity" : "Madison Beer", - "birthday" : "dt:1999-03-05 00:00:00", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "9 US", - "type" : "women", - "tags" : list, - }, - }), - ("https://www.wikifeetx.com/Tifa_Quinn", { - "pattern": (r"https://pics\.wikifeet\.com/Tifa_Quinn" - r"-Feet-\d+\.jpg"), - "count" : ">= 9", - "keyword": { - "celeb" : "Tifa_Quinn", - "celebrity" : "Tifa Quinn", - "birthday" : "[NOT SET]", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "4 US", - "type" : "women", - "tags" : list, - }, - }), - ("https://men.wikifeet.com/Chris_Hemsworth", { - "pattern": (r"https://pics\.wikifeet\.com/Chris_Hemsworth" - r"-Feet-\d+\.jpg"), - "count" : ">= 860", - "keyword": { - "celeb" : "Chris_Hemsworth", - "celebrity" : "Chris Hemsworth", - "birthday" : "dt:1983-08-11 00:00:00", - "birthplace": "Australia", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "12.5 US", - "type" : "men", - "tags" : list, - }, - }), - ) + example = "https://www.wikifeet.com/CELEB" def __init__(self, match): self.root = text.root_from_url(match.group(0)) diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index b308e74a..6dc9362a 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -32,54 +32,7 @@ class XhamsterGalleryExtractor(XhamsterExtractor): filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/photos/gallery/[^/?#]+)" - test = ( - ("https://xhamster.com/photos/gallery/11748968", { - "pattern": r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", - "count": ">= 144", - "keyword": { - "comments": int, - "count": int, - "favorite": bool, - "id": int, - "num": int, - "height": int, - "width": int, - "imageURL": str, - "pageURL": str, - "thumbURL": str, - "gallery": { - "date": "dt:2019-04-16 00:07:31", - "description": "", - "dislikes": int, - "id": 11748968, - "likes": int, - "tags": ["NON-Porn"], - "thumbnail": str, - "title": "Make the world better.", - "views": int, - }, - "user": { - "id": 16874672, - "name": "Anonymousrants", - "retired": bool, - "subscribers": int, - "url": "https://xhamster.com/users/anonymousrants", - "verified": bool, - }, - }, - }), - ("https://jp.xhamster2.com/photos/gallery/11748968", { - "pattern": r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", - "count": ">= 144", - }), - ("https://xhamster.com/photos/gallery/make-the-world-better-11748968"), - ("https://xhamster.com/photos/gallery/11748968"), - ("https://xhamster.one/photos/gallery/11748968"), - ("https://xhamster.desi/photos/gallery/11748968"), - ("https://xhamster2.com/photos/gallery/11748968"), - ("https://en.xhamster.com/photos/gallery/11748968"), - ("https://xhamster.porncache.net/photos/gallery/11748968"), - ) + example = "https://xhamster.com/photos/gallery/12345" def __init__(self, match): XhamsterExtractor.__init__(self, match) @@ -150,14 +103,7 @@ class XhamsterUserExtractor(XhamsterExtractor): """Extractor for all galleries of an xhamster user""" subcategory = "user" pattern = BASE_PATTERN + r"/users/([^/?#]+)(?:/photos)?/?(?:$|[?#])" - test = ( - ("https://xhamster.com/users/goldenpalomino/photos", { - "pattern": XhamsterGalleryExtractor.pattern, - "count": 50, - "range": "1-50", - }), - ("https://xhamster.com/users/nickname68"), - ) + example = "https://xhamster.com/users/USER/photos" def __init__(self, match): XhamsterExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py index 46ea0743..a28d8f5a 100644 --- a/gallery_dl/extractor/xvideos.py +++ b/gallery_dl/extractor/xvideos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -28,30 +28,7 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): pattern = (r"(?:https?://)?(?:www\.)?xvideos\.com" r"/(?:profiles|amateur-channels|model-channels)" r"/([^/?#]+)/photos/(\d+)") - test = ( - ("https://www.xvideos.com/profiles/pervertedcouple/photos/751031", { - "count": 8, - "pattern": r"https://profile-pics-cdn\d+\.xvideos-cdn\.com" - r"/[^/]+\,\d+/videos/profiles/galleries/84/ca/37" - r"/pervertedcouple/gal751031/pic_\d+_big\.jpg", - "keyword": { - "gallery": { - "id" : 751031, - "title": "Random Stuff", - "tags" : list, - }, - "user": { - "id" : 20245371, - "name" : "pervertedcouple", - "display" : "Pervertedcouple", - "sex" : "Woman", - "description": str, - }, - }, - }), - ("https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12"), - ("https://www.xvideos.com/model-channels/pervertedcouple/photos/12"), - ) + example = "https://www.xvideos.com/profiles/USER/photos/12345" def __init__(self, match): self.user, self.gallery_id = match.groups() @@ -97,13 +74,7 @@ class XvideosUserExtractor(XvideosBase, Extractor): categorytransfer = True pattern = (r"(?:https?://)?(?:www\.)?xvideos\.com" r"/profiles/([^/?#]+)/?(?:#.*)?$") - test = ( - ("https://www.xvideos.com/profiles/pervertedcouple", { - "url": "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", - "keyword": "335a3304941ff2e666c0201e9122819b61b34adb", - }), - ("https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos"), - ) + example = "https://www.xvideos.com/profiles/USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/ytdl.py b/gallery_dl/extractor/ytdl.py index 7f3c8de7..cb3c74c7 100644 --- a/gallery_dl/extractor/ytdl.py +++ b/gallery_dl/extractor/ytdl.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,7 @@ class YoutubeDLExtractor(Extractor): filename_fmt = "{title}-{id}.{extension}" archive_fmt = "{extractor_key} {id}" pattern = r"ytdl:(.*)" - test = ("ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9",) + example = "ytdl:https://www.youtube.com/watch?v=abcdefghijk" def __init__(self, match): # import main youtube_dl module diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index 8187db88..5fe1943c 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -111,23 +111,7 @@ class ZerochanTagExtractor(ZerochanExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?!\d+$)([^/?#]+)/?(?:\?([^#]+))?" - test = ("https://www.zerochan.net/Perth+%28Kantai+Collection%29", { - "pattern": r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "count": "> 24", - "keyword": { - "extension": r"re:jpg|png", - "file_url": r"re:https://static\.zerochan\.net" - r"/.+\.full\.\d+\.(jpg|png)", - "filename": r"re:(Perth\.\(Kantai\.Collection\)" - r"|Kantai\.Collection)\.full\.\d+", - "height": r"re:^\d+$", - "id": r"re:^\d+$", - "name": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "search_tags": "Perth (Kantai Collection)", - "size": r"re:^\d+k$", - "width": r"re:^\d+$", - }, - }) + example = "https://www.zerochan.net/TAG" def __init__(self, match): ZerochanExtractor.__init__(self, match) @@ -177,40 +161,7 @@ class ZerochanTagExtractor(ZerochanExtractor): class ZerochanImageExtractor(ZerochanExtractor): subcategory = "image" pattern = BASE_PATTERN + r"/(\d+)" - test = ("https://www.zerochan.net/2920445", { - "pattern": r"https://static\.zerochan\.net/" - r"Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "keyword": { - "author": "YeFan 葉凡", - "date": "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net" - "/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height": 1366, - "id": 2920445, - "path": ["Kantai Collection", "Perth (Kantai Collection)"], - "size": 1975296, - "tags": [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Blonde Hair", - "Theme:Braids", - "Theme:Coat", - "Theme:Female", - "Theme:Firefighter Outfit", - "Theme:Group", - "Theme:Long Sleeves", - "Theme:Personification", - "Theme:Pins", - "Theme:Ribbon", - "Theme:Shirt", - "Theme:Short Hair", - ], - "uploader": "YukinoTokisaki", - "width": 1920, - }, - }) + example = "https://www.zerochan.net/12345" def __init__(self, match): ZerochanExtractor.__init__(self, match) From 65b6011cc5225260d3c45c0149bbde0e14553898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 11 Sep 2023 17:20:06 +0200 Subject: [PATCH 35/53] update test/test_extractor.py --- test/results/35photo.py | 2 ++ test/results/generic.py | 16 ++++++++-------- test/results/horne.py | 2 +- test/results/nijie.py | 2 +- test/test_extractor.py | 25 ++++++++++++++----------- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/test/results/35photo.py b/test/results/35photo.py index b9c7673a..c0e29d53 100644 --- a/test/results/35photo.py +++ b/test/results/35photo.py @@ -19,6 +19,8 @@ __tests__ = ( { "#url" : "https://35photo.pro/suhoveev", + "#comment" : "last photo ID (1267028) isn't given as 'photo-id=\"<id>\" - " + "there are only 23 photos without the last one", "#category": ("", "35photo", "user"), "#class" : _35photo._35photoUserExtractor, "#count" : ">= 33", diff --git a/test/results/generic.py b/test/results/generic.py index dea9f7cd..5ec0400e 100644 --- a/test/results/generic.py +++ b/test/results/generic.py @@ -9,7 +9,7 @@ from gallery_dl.extractor import generic __tests__ = ( { - "#url" : "https://www.nongnu.org/lzip/", + "#url" : "generic:https://www.nongnu.org/lzip/", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, "#count" : 1, @@ -22,7 +22,7 @@ __tests__ = ( }, { - "#url" : "https://räksmörgås.josefsson.org/", + "#url" : "generic:https://räksmörgås.josefsson.org/", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, "#pattern" : "^https://räksmörgås.josefsson.org/", @@ -30,37 +30,37 @@ __tests__ = ( }, { - "#url" : "https://en.wikipedia.org/Main_Page", + "#url" : "generic:https://en.wikipedia.org/Main_Page", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, { - "#url" : "https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, { - "#url" : "https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, { - "#url" : "https://en.wikipedia.org/Main_Page", + "#url" : "generic:https://en.wikipedia.org/Main_Page", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, { - "#url" : "https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, { - "#url" : "https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", "#category": ("", "generic", ""), "#class" : generic.GenericExtractor, }, diff --git a/test/results/horne.py b/test/results/horne.py index eb7b0a79..1cea3a0e 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -68,7 +68,7 @@ __tests__ = ( { "#url" : "https://horne.red/like_my.php", "#category": ("Nijie", "horne", "followed"), - "#class" : nijie.NijiefollowedExtractor, + "#class" : nijie.NijieFollowedExtractor, }, { diff --git a/test/results/nijie.py b/test/results/nijie.py index b2b10a70..2b5785af 100644 --- a/test/results/nijie.py +++ b/test/results/nijie.py @@ -86,7 +86,7 @@ __tests__ = ( { "#url" : "https://nijie.info/like_my.php", "#category": ("Nijie", "nijie", "followed"), - "#class" : nijie.NijiefollowedExtractor, + "#class" : nijie.NijieFollowedExtractor, }, { diff --git a/test/test_extractor.py b/test/test_extractor.py index fb164af4..41f47394 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -93,20 +93,24 @@ class TestExtractorModule(unittest.TestCase): FakeExtractor.from_url(invalid) def test_unique_pattern_matches(self): - test_urls = [] + try: + import test.results + except ImportError: + raise unittest.SkipTest("no test data") # collect testcase URLs + test_urls = [] append = test_urls.append - for extr in extractor.extractors(): - for testcase in extr._get_tests(): - append((testcase[0], extr)) + + for result in test.results.all(): + append((result["#url"], result["#class"])) # iterate over all testcase URLs for url, extr1 in test_urls: matches = [] # ... and apply all regex patterns to each one - for extr2 in extractor._cache: + for extr2 in _list_classes(): # skip DirectlinkExtractor pattern if it isn't tested if extr1 != DirectlinkExtractor and \ @@ -133,15 +137,14 @@ class TestExtractorModule(unittest.TestCase): self.assertIs(extr1, matches[0][1], url) def test_init(self): - """Test for exceptions in Extractor.initialize(()""" + """Test for exceptions in Extractor.initialize() and .finalize()""" for cls in extractor.extractors(): if cls.category == "ytdl": continue - for test in cls._get_tests(): - extr = cls.from_url(test[0]) - extr.initialize() - extr.finalize() - break + extr = cls.from_url(cls.example) + extr.initialize() + extr.finalize() + break @unittest.skipIf(sys.hexversion < 0x3060000, "test fails in CI") def test_init_ytdl(self): From 4cdab8074e09e349824ee1cbec60329bae21378b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 11 Sep 2023 17:32:59 +0200 Subject: [PATCH 36/53] update/fix --list-extractors --- gallery_dl/__init__.py | 11 +++++------ gallery_dl/extractor/common.py | 16 ---------------- gallery_dl/extractor/flickr.py | 13 +------------ 3 files changed, 6 insertions(+), 34 deletions(-) diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 62622b9a..d3a0f588 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -196,16 +196,15 @@ def main(): elif args.list_extractors: write = sys.stdout.write - fmt = "{}\n{}\nCategory: {} - Subcategory: {}{}\n\n".format + fmt = ("{}{}\nCategory: {} - Subcategory: {}" + "\nExample : {}\n\n").format for extr in extractor.extractors(): - if not extr.__doc__: - continue - test = next(extr._get_tests(), None) write(fmt( - extr.__name__, extr.__doc__, + extr.__name__, + "\n" + extr.__doc__ if extr.__doc__ else "", extr.category, extr.subcategory, - "\nExample : " + test[0] if test else "", + extr.example, )) elif args.clear_cache: diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 61861dff..399e3e24 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -34,7 +34,6 @@ class Extractor(): archive_fmt = "" cookies_domain = "" browser = None - example = "" root = "" request_interval = 0.0 request_interval_min = 0.0 @@ -508,21 +507,6 @@ class Extractor(): result.append((Message.Queue, url, {"_extractor": extr})) return iter(result) - @classmethod - def _get_tests(cls): - """Yield an extractor's test cases as (URL, RESULTS) tuples""" - tests = cls.test - if not tests: - return - - if len(tests) == 2 and (not tests[1] or isinstance(tests[1], dict)): - tests = (tests,) - - for test in tests: - if isinstance(test, str): - test = (test, None) - yield test - @classmethod def _dump(cls, obj): util.dump_json(obj, ensure_ascii=False, indent=2) diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py index b48c035a..ea327654 100644 --- a/gallery_dl/extractor/flickr.py +++ b/gallery_dl/extractor/flickr.py @@ -110,18 +110,7 @@ class FlickrAlbumExtractor(FlickrExtractor): "Albums", "{album[id]} {album[title]}") archive_fmt = "a_{album[id]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/(?:album|set)s(?:/(\d+))?" - test = ( - (("https://www.flickr.com/photos/shona_s/albums/72157633471741607"), { - "pattern": FlickrImageExtractor.pattern, - "count": 6, - }), - ("https://www.flickr.com/photos/shona_s/albums", { - "pattern": pattern, - "count": 2, - }), - ("https://secure.flickr.com/photos/shona_s/albums"), - ("https://m.flickr.com/photos/shona_s/albums"), - ) + example = "https://www.flickr.com/photos/USER/albums/12345" def __init__(self, match): FlickrExtractor.__init__(self, match) From d3bd163e7fe03643238fcbbcfd6191b0c267752a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 11 Sep 2023 17:36:44 +0200 Subject: [PATCH 37/53] update scripts/supportedsites.py --- scripts/supportedsites.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index b8052164..59d15192 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -14,6 +14,11 @@ import collections import util from gallery_dl import extractor +try: + from test import results +except ImportError: + results = None + CATEGORY_MAP = { "2chan" : "Futaba Channel", @@ -367,17 +372,8 @@ def domain(cls): if hasattr(cls, "root") and cls.root: return cls.root + "/" - if hasattr(cls, "https"): - scheme = "https" if cls.https else "http" - netloc = cls.__doc__.split()[-1] - return "{}://{}/".format(scheme, netloc) - - test = next(cls._get_tests(), None) - if test: - url = test[0] - return url[:url.find("/", 8)+1] - - return "" + url = cls.example + return url[:url.find("/", 8)+1] def category_text(c): @@ -432,14 +428,10 @@ def build_extractor_list(): for category, root in extr.instances: base[category].append(extr.subcategory) if category not in domains: - if not root: + if not root and results: # use domain from first matching test - for url, _ in extr._get_tests(): - if extr.from_url(url).category == category: - root = url[:url.index("/", 8)] - break - else: - continue + test = results.category(category)[0] + root = test["#class"].from_url(test["#url"]).root domains[category] = root + "/" # sort subcategory lists From 1710f1e98318161290398dba26528c34c4d484cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Tue, 12 Sep 2023 21:41:31 +0200 Subject: [PATCH 38/53] [reddit] improve comment metadata (#4482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - provide 'date' - make metadata of the main submission available as 'submission[…]' --- gallery_dl/extractor/reddit.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 36708f1a..c02e790e 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -97,9 +97,13 @@ class RedditExtractor(Extractor): ' href="', '"'): urls.append((url, submission)) for comment in comments: - for url in text.extract_iter( - comment["body_html"] or "", ' href="', '"'): - urls.append((url, comment)) + html = comment["body_html"] or "" + if ' href="' in html: + comment["submission"] = submission + comment["date"] = text.parse_timestamp( + comment["created_utc"]) + for url in text.extract_iter(html, ' href="', '"'): + urls.append((url, comment)) for url, data in urls: if not url or url[0] == "#": From ae5e049c4f8f70c981d90794cbfd02b29580c733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Tue, 12 Sep 2023 20:27:55 +0200 Subject: [PATCH 39/53] [redgifs] provide 'collection' metadata in a separate field (#4508) instead of overwriting the actual metadata --- gallery_dl/extractor/redgifs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index abd21b30..167877ce 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -111,8 +111,9 @@ class RedgifsUserExtractor(RedgifsExtractor): class RedgifsCollectionExtractor(RedgifsExtractor): """Extractor for an individual user collection""" subcategory = "collection" - directory_fmt = ("{category}", "{userName}", "{folderName}") - archive_fmt = "{folderId}_{id}" + directory_fmt = ( + "{category}", "{collection[userName]}", "{collection[folderName]}") + archive_fmt = "{collection[folderId]}_{id}" pattern = (r"(?:https?://)?(?:www\.)?redgifs\.com/users" r"/([^/?#]+)/collections/([^/?#]+)") test = ( @@ -133,9 +134,9 @@ class RedgifsCollectionExtractor(RedgifsExtractor): self.collection_id = match.group(2) def metadata(self): - data = {"userName": self.key} - data.update(self.api.collection_info(self.key, self.collection_id)) - return data + collection = self.api.collection_info(self.key, self.collection_id) + collection["userName"] = self.key + return {"collection": collection} def gifs(self): return self.api.collection(self.key, self.collection_id) From 32da3c70d3153568eb9aaf5a71ab2875e7767850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Tue, 12 Sep 2023 20:53:35 +0200 Subject: [PATCH 40/53] [behance] handle videos without 'renditions' (#4523) --- gallery_dl/extractor/behance.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index cf123d91..90e5660e 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -177,7 +177,13 @@ class BehanceGalleryExtractor(BehanceExtractor): append((url, module)) elif mtype == "VideoModule": - renditions = module["videoData"]["renditions"] + try: + renditions = module["videoData"]["renditions"] + except Exception: + self.log.warning("No download URLs for video %s", + module.get("id") or "???") + continue + try: url = [ r["url"] for r in renditions @@ -186,6 +192,7 @@ class BehanceGalleryExtractor(BehanceExtractor): except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) url = "ytdl:" + renditions[-1]["url"] + append((url, module)) elif mtype == "MediaCollectionModule": From ac00d47a168bccf3c1249d939a091f9c02f1ae51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 13 Sep 2023 14:54:25 +0200 Subject: [PATCH 41/53] update test/test_results.py --- test/test_results.py | 134 ++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/test/test_results.py b/test/test_results.py index a555b721..8644f275 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -15,10 +15,12 @@ import re import json import hashlib import datetime +import collections sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import \ extractor, util, job, config, exception, formatter # noqa E402 +from test import results # noqa E402 # temporary issues, etc. @@ -46,28 +48,33 @@ class TestExtractorResults(unittest.TestCase): for url, exc in cls._skipped: print('- {} ("{}")'.format(url, exc)) - def _run_test(self, extr, url, result): - if result: - if "options" in result: - for key, value in result["options"]: + def _run_test(self, result): + result.pop("#comment", None) + only_matching = (len(result) <= 3) + + if only_matching: + content = False + else: + if "#options" in result: + for key, value in result["#options"].items(): key = key.split(".") config.set(key[:-1], key[-1], value) - if "range" in result: - config.set((), "image-range" , result["range"]) - config.set((), "chapter-range", result["range"]) - content = "content" in result - else: - content = False + if "#range" in result: + config.set((), "image-range" , result["#range"]) + config.set((), "chapter-range", result["#range"]) + content = ("#sha1_content" in result) - tjob = ResultJob(url, content=content) - self.assertEqual(extr, tjob.extractor.__class__) + tjob = ResultJob(result["#url"], content=content) + self.assertEqual(result["#class"], tjob.extractor.__class__) - if not result: + if only_matching: return - if "exception" in result: - with self.assertRaises(result["exception"]): + + if "#exception" in result: + with self.assertRaises(result["#exception"]): tjob.run() return + try: tjob.run() except exception.StopExtraction: @@ -76,11 +83,11 @@ class TestExtractorResults(unittest.TestCase): exc = str(exc) if re.match(r"'5\d\d ", exc) or \ re.search(r"\bRead timed out\b", exc): - self._skipped.append((url, exc)) + self._skipped.append((result["#url"], exc)) self.skipTest(exc) raise - if result.get("archive", True): + if result.get("#archive", True): self.assertEqual( len(set(tjob.archive_list)), len(tjob.archive_list), @@ -92,7 +99,7 @@ class TestExtractorResults(unittest.TestCase): for url, kwdict in zip(tjob.url_list, tjob.kwdict_list): if "_extractor" in kwdict: extr = kwdict["_extractor"].from_url(url) - if extr is None and not result.get("extractor", True): + if extr is None and not result.get("#extractor", True): continue self.assertIsInstance(extr, kwdict["_extractor"]) self.assertEqual(extr.url, url) @@ -102,27 +109,26 @@ class TestExtractorResults(unittest.TestCase): self.assertIn("extension", kwdict) # test extraction results - if "url" in result: - self.assertEqual(result["url"], tjob.url_hash.hexdigest()) + if "#sha1_url" in result: + self.assertEqual( + result["#sha1_url"], tjob.url_hash.hexdigest()) - if "content" in result: - expected = result["content"] + if "#sha1_content" in result: + expected = result["#sha1_content"] digest = tjob.content_hash.hexdigest() if isinstance(expected, str): - self.assertEqual(digest, expected, "content") - else: # assume iterable - self.assertIn(digest, expected, "content") - - if "keyword" in result: - expected = result["keyword"] - if isinstance(expected, dict): - for kwdict in tjob.kwdict_list: - self._test_kwdict(kwdict, expected) - else: # assume SHA1 hash - self.assertEqual(expected, tjob.kwdict_hash.hexdigest()) - - if "count" in result: - count = result["count"] + self.assertEqual( + expected, digest, "content") + else: # iterable + self.assertIn( + digest, expected, "content") + + if "#sha1_metadata" in result: + self.assertEqual( + result["#sha1_metadata"], tjob.kwdict_hash.hexdigest()) + + if "#count" in result: + count = result["#count"] if isinstance(count, str): self.assertRegex(count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$") expr = "{} {}".format(len(tjob.url_list), count) @@ -130,10 +136,15 @@ class TestExtractorResults(unittest.TestCase): else: # assume integer self.assertEqual(len(tjob.url_list), count) - if "pattern" in result: + if "#pattern" in result: self.assertGreater(len(tjob.url_list), 0) for url in tjob.url_list: - self.assertRegex(url, result["pattern"]) + self.assertRegex(url, result["#pattern"]) + + metadata = {k: v for k, v in result.items() if k[0] != "#"} + if metadata: + for kwdict in tjob.kwdict_list: + self._test_kwdict(kwdict, metadata) def _test_kwdict(self, kwdict, tests): for key, test in tests.items(): @@ -352,39 +363,34 @@ def setup_test_config(): def generate_tests(): """Dynamically generate extractor unittests""" - def _generate_test(extr, tcase): + def _generate_method(result): def test(self): - url, result = tcase - print("\n", url, sep="") - self._run_test(extr, url, result) + print("\n" + result["#url"]) + self._run_test(result) return test # enable selective testing for direct calls - if __name__ == '__main__' and len(sys.argv) > 1: - categories = sys.argv[1:] - negate = False - if categories[0].lower() == "all": - categories = () - negate = True - elif categories[0].lower() == "broken": - categories = BROKEN + if __name__ == "__main__" and len(sys.argv) > 1: + category, _, subcategory = sys.argv[1].partition(":") del sys.argv[1:] + + tests = results.category(category) + if subcategory: + tests = [t for t in tests if t["#category"][-1] == subcategory] else: - categories = BROKEN - negate = True - if categories: - print("skipping:", ", ".join(categories)) - fltr = util.build_extractor_filter(categories, negate=negate) + tests = results.all() # add 'test_...' methods - for extr in filter(fltr, extractor.extractors()): - name = "test_" + extr.__name__ + "_" - for num, tcase in enumerate(extr._get_tests(), 1): - test = _generate_test(extr, tcase) - test.__name__ = name + str(num) - setattr(TestExtractorResults, test.__name__, test) + enum = collections.defaultdict(int) + for result in tests: + name = "{1}_{2}".format(*result["#category"]) + enum[name] += 1 + + method = _generate_method(result) + method.__name__ = "test_{}_{}".format(name, enum[name]) + setattr(TestExtractorResults, method.__name__, method) generate_tests() -if __name__ == '__main__': - unittest.main(warnings='ignore') +if __name__ == "__main__": + unittest.main(warnings="ignore") From 899df8f2377f284d29baf90cddacda4aff81e0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 13 Sep 2023 21:38:31 +0200 Subject: [PATCH 42/53] remove another '*' for keyword-only arguments 076380e0 --- gallery_dl/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gallery_dl/util.py b/gallery_dl/util.py index bf67a64d..62e7b4aa 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -239,8 +239,7 @@ def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): fp.write("\n") -def dump_response(response, fp, *, - headers=False, content=True, hide_auth=True): +def dump_response(response, fp, headers=False, content=True, hide_auth=True): """Write the contents of 'response' into a file-like object""" if headers: From 9f75713e00e603bcba3e8749eb5fa5987233ca56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 13 Sep 2023 21:47:20 +0200 Subject: [PATCH 43/53] [recursive] simplify --- gallery_dl/extractor/recursive.py | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/gallery_dl/extractor/recursive.py b/gallery_dl/extractor/recursive.py index da1f5eac..41564842 100644 --- a/gallery_dl/extractor/recursive.py +++ b/gallery_dl/extractor/recursive.py @@ -9,7 +9,6 @@ """Recursive extractor""" from .common import Extractor, Message -import requests import re @@ -20,29 +19,13 @@ class RecursiveExtractor(Extractor): example = "recursive:https://pastebin.com/raw/FLwrCYsT" def items(self): - self.session.mount("file://", FileAdapter()) - page = self.request(self.url.partition(":")[2]).text - del self.session.adapters["file://"] + url = self.url.partition(":")[2] - for match in re.finditer(r"https?://[^\s\"']+", page): - yield Message.Queue, match.group(0), {} - - -class FileAdapter(requests.adapters.BaseAdapter): - """Requests adapter for local files""" - - def send(self, request, **kwargs): - response = requests.Response() - try: - response.raw = open(request.url[7:], "rb") - except OSError: - import io - response.raw = io.BytesIO() - response.status_code = requests.codes.bad_request + if url.startswith("file://"): + with open(url[7:]) as fp: + page = fp.read() else: - response.raw.release_conn = response.raw.close - response.status_code = requests.codes.ok - return response + page = self.request(url).text - def close(self): - pass + for match in re.finditer(r"https?://[^\s\"']+", page): + yield Message.Queue, match.group(0), {} From 24a1d4639134358b3ee1b229684b14e0acdf3d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Wed, 13 Sep 2023 23:42:51 +0200 Subject: [PATCH 44/53] [mastodon] support '/@USER/following' URLs Previously, only '/users/USER/following' got matched. --- gallery_dl/extractor/mastodon.py | 4 ++-- test/results/mastodonsocial.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index cb94481e..3c2b03ee 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -131,8 +131,8 @@ class MastodonBookmarkExtractor(MastodonExtractor): class MastodonFollowingExtractor(MastodonExtractor): """Extractor for followed mastodon users""" subcategory = "following" - pattern = BASE_PATTERN + r"/users/([^/?#]+)/following" - example = "https://mastodon.social/users/USER/following" + pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)/following" + example = "https://mastodon.social/@USER/following" def items(self): api = MastodonAPI(self) diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py index a7001fd0..2ad0dc20 100644 --- a/test/results/mastodonsocial.py +++ b/test/results/mastodonsocial.py @@ -70,6 +70,12 @@ __tests__ = ( "#extractor": False, }, +{ + "#url" : "https://mastodon.social/@0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + { "#url" : "https://mastodon.social/users/id:10843/following", "#category": ("mastodon", "mastodon.social", "following"), From 27ec653991a611ffc6f42ae3af03b81ce5489172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Thu, 14 Sep 2023 13:27:03 +0200 Subject: [PATCH 45/53] fix bug in test_init and update example URLs --- gallery_dl/extractor/blogger.py | 2 +- gallery_dl/extractor/comicvine.py | 2 +- gallery_dl/extractor/fapello.py | 2 +- gallery_dl/extractor/foolslide.py | 2 +- gallery_dl/extractor/gelbooru.py | 7 ++++--- gallery_dl/extractor/gelbooru_v01.py | 2 +- gallery_dl/extractor/hentaifoundry.py | 2 +- gallery_dl/extractor/imagehosts.py | 4 ++-- gallery_dl/extractor/inkbunny.py | 4 ++-- gallery_dl/extractor/instagram.py | 2 +- gallery_dl/extractor/kemonoparty.py | 4 ++-- gallery_dl/extractor/komikcast.py | 2 +- gallery_dl/extractor/mangapark.py | 2 +- gallery_dl/extractor/newgrounds.py | 2 +- gallery_dl/extractor/reddit.py | 2 +- gallery_dl/extractor/webmshare.py | 2 +- test/test_extractor.py | 1 - 17 files changed, 22 insertions(+), 22 deletions(-) diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index 4f767e0a..d75c3498 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -96,7 +96,7 @@ class BloggerPostExtractor(BloggerExtractor): """Extractor for a single blog post""" subcategory = "post" pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)" - example = "https://BLOG.blogspot.com/YYYY/MM/TITLE.html" + example = "https://BLOG.blogspot.com/1970/01/TITLE.html" def __init__(self, match): BloggerExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/comicvine.py b/gallery_dl/extractor/comicvine.py index 36dc7421..d076795c 100644 --- a/gallery_dl/extractor/comicvine.py +++ b/gallery_dl/extractor/comicvine.py @@ -25,7 +25,7 @@ class ComicvineTagExtractor(BooruExtractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?comicvine\.gamespot\.com" r"(/([^/?#]+)/(\d+-\d+)/images/.*)") - example = "https://comicvine.gamespot.com/TAG/ID/images/" + example = "https://comicvine.gamespot.com/TAG/123-45/images/" def __init__(self, match): BooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/fapello.py b/gallery_dl/extractor/fapello.py index 5329a17c..d4524e0e 100644 --- a/gallery_dl/extractor/fapello.py +++ b/gallery_dl/extractor/fapello.py @@ -80,7 +80,7 @@ class FapelloPathExtractor(Extractor): pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" r"|popular_videos/[^/?#]+)/?$") - example = "https://fapello.com/PATH/" + example = "https://fapello.com/trending/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 6db207a1..b0699b03 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -53,7 +53,7 @@ class FoolslideChapterExtractor(FoolslideExtractor): "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" - example = "https://read.powermanga.org/read/MANGA/en/0/CHAPTER/" + example = "https://read.powermanga.org/read/MANGA/en/0/123/" def items(self): page = self.request(self.gallery_url).text diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index d8528c96..b62ff78b 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -215,9 +215,10 @@ class GelbooruRedirectExtractor(GelbooruBase, Extractor): def __init__(self, match): Extractor.__init__(self, match) - self.redirect_url = text.ensure_http_scheme( - binascii.a2b_base64(match.group(1)).decode()) + self.url_base64 = match.group(1) def items(self): + url = text.ensure_http_scheme(binascii.a2b_base64( + self.url_base64).decode()) data = {"_extractor": GelbooruPostExtractor} - yield Message.Queue, self.redirect_url, data + yield Message.Queue, url, data diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index 9cdf904a..0b96048b 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -111,7 +111,7 @@ class GelbooruV01FavoriteExtractor(GelbooruV01Extractor): archive_fmt = "f_{favorite_id}_{id}" per_page = 50 pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" - example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=N" + example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=1" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py index d4670eb9..4c02000c 100644 --- a/gallery_dl/extractor/hentaifoundry.py +++ b/gallery_dl/extractor/hentaifoundry.py @@ -232,7 +232,7 @@ class HentaifoundryRecentExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "Recent Pictures", "{date}") archive_fmt = "r_{index}" pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)" - example = "https://www.hentai-foundry.com/pictures/recent/YYYY-MM-DD" + example = "https://www.hentai-foundry.com/pictures/recent/1970-01-01" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index ddd4c888..94019bd6 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -246,8 +246,8 @@ class PostimgGalleryExtractor(ImagehostImageExtractor): """Extractor for images galleries from postimages.org""" category = "postimg" subcategory = "gallery" - pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" - r"/(?:gallery/)([^/?#]+)/?)") + pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" + r"\.(?:cc|org)/gallery/([^/?#]+))") example = "https://postimages.org/gallery/ID" def items(self): diff --git a/gallery_dl/extractor/inkbunny.py b/gallery_dl/extractor/inkbunny.py index 1a4cf21f..4ad37fc1 100644 --- a/gallery_dl/extractor/inkbunny.py +++ b/gallery_dl/extractor/inkbunny.py @@ -135,7 +135,7 @@ class InkbunnyFavoriteExtractor(InkbunnyExtractor): r"userfavorites_process\.php\?favs_user_id=(\d+)|" r"submissionsviewall\.php\?([^#]+&mode=userfavs&[^#]+))") example = ("https://inkbunny.net/userfavorites_process.php" - "?mode=userfavs&favs_user_id=12345") + "?favs_user_id=12345") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -167,7 +167,7 @@ class InkbunnySearchExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/submissionsviewall\.php\?([^#]+&mode=search&[^#]+)") example = ("https://inkbunny.net/submissionsviewall.php" - "?mode=search&text=TAG") + "?text=TAG&mode=search&type=") def __init__(self, match): InkbunnyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index 10a1c7bf..e9631909 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -469,7 +469,7 @@ class InstagramGuideExtractor(InstagramExtractor): """Extractor for an Instagram guide""" subcategory = "guide" pattern = USER_PATTERN + r"/guide/[^/?#]+/(\d+)" - example = "https://www.instagram.com/USER/guide/" + example = "https://www.instagram.com/USER/guide/NAME/12345" def __init__(self, match): InstagramExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 0034e690..54cd99cd 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -267,7 +267,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): filename_fmt = "{id}_{num:>02}_{filename}.{extension}" archive_fmt = "discord_{server}_{id}_{num}" pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)" - example = "https://kemono.party/discard/server/12345/channel/12345" + example = "https://kemono.party/discord/server/12345#CHANNEL" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -346,7 +346,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): class KemonopartyDiscordServerExtractor(KemonopartyExtractor): subcategory = "discord-server" pattern = BASE_PATTERN + r"/discord/server/(\d+)$" - example = "https://kemono.party/discard/server/12345" + example = "https://kemono.party/discord/server/12345" def __init__(self, match): KemonopartyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/komikcast.py b/gallery_dl/extractor/komikcast.py index 635b5699..a3e01305 100644 --- a/gallery_dl/extractor/komikcast.py +++ b/gallery_dl/extractor/komikcast.py @@ -48,7 +48,7 @@ class KomikcastBase(): class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor): """Extractor for manga-chapters from komikcast.site""" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/)" - example = "https://komikcast.site/chapter/TITLE" + example = "https://komikcast.site/chapter/TITLE/" def metadata(self, page): info = text.extr(page, "<title>", " - Komikcast<") diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index 6b8f869a..63aaf917 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -83,7 +83,7 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor): """Extractor for manga from mangapark.net""" subcategory = "manga" pattern = BASE_PATTERN + r"/title/(\d+)(?:-[^/?#]*)?/?$" - example = "https://mangapark.net/title/MANGA" + example = "https://mangapark.net/title/12345-MANGA" def __init__(self, match): self.root = text.root_from_url(match.group(0)) diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 18c1e5c3..55f5c285 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -349,7 +349,7 @@ class NewgroundsMoviesExtractor(NewgroundsExtractor): """Extractor for all movies of a newgrounds user""" subcategory = _path = "movies" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$" - example = "https://USER.newgrounds.com/movies<" + example = "https://USER.newgrounds.com/movies" class NewgroundsGamesExtractor(NewgroundsExtractor): diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 2f169d59..e26b741c 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -244,7 +244,7 @@ class RedditSubmissionExtractor(RedditExtractor): pattern = (r"(?:https?://)?(?:" r"(?:\w+\.)?reddit\.com/(?:(?:r|u|user)/[^/?#]+" r"/comments|gallery)|redd\.it)/([a-z0-9]+)") - example = "https://www.reddit.com/r/SUBREDDIT/comments/ID/" + example = "https://www.reddit.com/r/SUBREDDIT/comments/id/" def __init__(self, match): RedditExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index a09b7d04..7e2b5ea3 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -21,7 +21,7 @@ class WebmshareVideoExtractor(Extractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?(?:s\d+\.)?webmshare\.com" r"/(?:play/|download-webm/)?(\w{3,})") - example = "https://webmshare.com/ID" + example = "https://webmshare.com/_ID_" def __init__(self, match): Extractor.__init__(self, match) diff --git a/test/test_extractor.py b/test/test_extractor.py index 41f47394..9387f5b6 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -144,7 +144,6 @@ class TestExtractorModule(unittest.TestCase): extr = cls.from_url(cls.example) extr.initialize() extr.finalize() - break @unittest.skipIf(sys.hexversion < 0x3060000, "test fails in CI") def test_init_ytdl(self): From d13c82eff1e533bcf65569a2c0fab9ccc74f5360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Thu, 14 Sep 2023 14:57:01 +0200 Subject: [PATCH 46/53] [kemonoparty] update favorites API endpoint (#4522) --- gallery_dl/extractor/kemonoparty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 54cd99cd..0c7d220c 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -382,7 +382,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): if self.favorites == "artist": users = self.request( - self.root + "/api/favorites?type=artist").json() + self.root + "/api/v1/account/favorites?type=artist").json() for user in users: user["_extractor"] = KemonopartyUserExtractor url = "{}/{}/user/{}".format( @@ -391,7 +391,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): elif self.favorites == "post": posts = self.request( - self.root + "/api/favorites?type=post").json() + self.root + "/api/v1/account/favorites?type=post").json() for post in posts: post["_extractor"] = KemonopartyPostExtractor url = "{}/{}/user/{}/post/{}".format( From 05f183d7d4772d1f2a3d5d9e71fee60d8af95110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Thu, 14 Sep 2023 18:49:58 +0200 Subject: [PATCH 47/53] exclude export_tests.py from linting for Python<3.8 --- .github/workflows/tests.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36ab9534..510045a0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,7 +53,14 @@ jobs: - name: Lint with flake8 run: | - flake8 . + case "${{ matrix.python-version }}" in + 3.4|3.5|3.6|3.7) + flake8 --extend-exclude scripts/export_tests.py . + ;; + *) + flake8 . + ;; + esac - name: Run tests run: | From fdcfae347f710443dcb1a8f28234738209056b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Fri, 15 Sep 2023 14:19:47 +0200 Subject: [PATCH 48/53] do not restrict flake8 version --- .github/workflows/tests.yml | 3 +-- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 510045a0..168edb9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,8 +32,7 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - pip install "flake8<4" "importlib-metadata<5" - pip install youtube-dl + pip install flake8 youtube-dl - name: Install yt-dlp run: | diff --git a/setup.cfg b/setup.cfg index a66d7cf3..e115e874 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -exclude = .git,__pycache__,build,dist,archive,test/results +exclude = .git,__pycache__,build,dist,archive,results ignore = E203,E226,W504 per-file-ignores = setup.py: E501 From 6178177227fe12c35129c9fd10c2baf4969ecfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Fri, 15 Sep 2023 23:04:30 +0200 Subject: [PATCH 49/53] [twitter] fix '_extractor' of following results (#4536) regression from 20ed647f --- docs/configuration.rst | 5 +++-- docs/gallery-dl.conf | 2 +- gallery_dl/extractor/twitter.py | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 819dc959..36d57fb8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3328,7 +3328,7 @@ extractor.twitter.users Type ``string`` Default - ``"timeline"`` + ``"user"`` Example ``"https://twitter.com/search?q=from:{legacy[screen_name]}"`` Description @@ -3339,7 +3339,8 @@ Description Special values: - * ``"timeline"``: ``https://twitter.com/i/user/{rest_id}`` + * ``"user"``: ``https://twitter.com/i/user/{rest_id}`` + * ``"timeline"``: ``https://twitter.com/id:{rest_id}/timeline`` * ``"tweets"``: ``https://twitter.com/id:{rest_id}/tweets`` * ``"media"``: ``https://twitter.com/id:{rest_id}/media`` diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index e0594cad..6345ded7 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -335,7 +335,7 @@ "text-tweets": false, "twitpic": false, "unique": true, - "users": "timeline", + "users": "user", "videos": true }, "unsplash": diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index d8890099..3895c74c 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -406,9 +406,12 @@ class TwitterExtractor(Extractor): def _users_result(self, users): userfmt = self.config("users") - if not userfmt or userfmt == "timeline": - cls = TwitterTimelineExtractor + if not userfmt or userfmt == "user": + cls = TwitterUserExtractor fmt = (self.root + "/i/user/{rest_id}").format_map + elif userfmt == "timeline": + cls = TwitterTimelineExtractor + fmt = (self.root + "/id:{rest_id}/timeline").format_map elif userfmt == "media": cls = TwitterMediaExtractor fmt = (self.root + "/id:{rest_id}/media").format_map From cb4798f07afea7eacf4a3427c3573f536f9dce5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Sun, 17 Sep 2023 15:19:19 +0200 Subject: [PATCH 50/53] [architizer] fix extraction (#4537) --- gallery_dl/extractor/architizer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py index 3d1fd138..8064e789 100644 --- a/gallery_dl/extractor/architizer.py +++ b/gallery_dl/extractor/architizer.py @@ -29,11 +29,13 @@ class ArchitizerProjectExtractor(GalleryExtractor): def metadata(self, page): extr = text.extract_from(page) + extr('id="Pages"', "") + return { - "title" : extr("data-name='", "'"), - "slug" : extr("data-slug='", "'"), - "gid" : extr("data-gid='", "'").rpartition(".")[2], - "firm" : extr("data-firm-leaders-str='", "'"), + "title" : extr('data-name="', '"'), + "slug" : extr('data-slug="', '"'), + "gid" : extr('data-gid="', '"').rpartition(".")[2], + "firm" : extr('data-firm-leaders-str="', '"'), "location" : extr("<h2>", "<").strip(), "type" : text.unescape(text.remove_html(extr( '<div class="title">Type</div>', '<br'))), @@ -52,7 +54,7 @@ class ArchitizerProjectExtractor(GalleryExtractor): return [ (url, None) for url in text.extract_iter( - page, "property='og:image:secure_url' content='", "?") + page, 'property="og:image:secure_url" content="', "?") ] From 3ecb51272293a1716f9ee09ec5f7229a27d7038b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de> Date: Mon, 18 Sep 2023 23:50:25 +0200 Subject: [PATCH 51/53] send Referer headers by default --- docs/configuration.rst | 18 +++++++++++++++++- gallery_dl/extractor/500px.py | 3 --- gallery_dl/extractor/8chan.py | 3 --- gallery_dl/extractor/artstation.py | 2 -- gallery_dl/extractor/behance.py | 5 ++--- gallery_dl/extractor/bunkr.py | 3 +-- gallery_dl/extractor/common.py | 7 +++++++ gallery_dl/extractor/deviantart.py | 4 +--- gallery_dl/extractor/exhentai.py | 1 - gallery_dl/extractor/fantia.py | 5 +---- gallery_dl/extractor/foolfuuka.py | 3 --- gallery_dl/extractor/hiperdex.py | 3 --- gallery_dl/extractor/hotleak.py | 3 --- gallery_dl/extractor/imagefap.py | 3 --- gallery_dl/extractor/imgur.py | 6 +----- gallery_dl/extractor/itaku.py | 1 - gallery_dl/extractor/kemonoparty.py | 1 - gallery_dl/extractor/mangafox.py | 3 --- gallery_dl/extractor/mangakakalot.py | 3 --- gallery_dl/extractor/manganelo.py | 2 -- gallery_dl/extractor/naverwebtoon.py | 1 - gallery_dl/extractor/newgrounds.py | 2 -- gallery_dl/extractor/nijie.py | 1 - gallery_dl/extractor/nozomi.py | 6 +++--- gallery_dl/extractor/patreon.py | 1 - gallery_dl/extractor/pinterest.py | 1 - gallery_dl/extractor/pornpics.py | 3 --- gallery_dl/extractor/reactor.py | 1 - gallery_dl/extractor/redgifs.py | 1 - gallery_dl/extractor/sankaku.py | 1 - gallery_dl/extractor/skeb.py | 7 +++---- gallery_dl/extractor/vipergirls.py | 3 --- gallery_dl/extractor/weibo.py | 1 - 33 files changed, 36 insertions(+), 72 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 36d57fb8..ee1567a9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -565,6 +565,21 @@ Description browser would use HTTP/2. +extractor.*.referer +------------------- +Type + * ``bool`` + * ``string`` +Default + ``true`` +Description + Send `Referer <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referer>`__ + headers with all outgoing HTTP requests. + + If this is a ``string``, send it as Referer + instead of the extractor's ``root`` domain. + + extractor.*.headers ------------------- Type @@ -576,7 +591,8 @@ Default "User-Agent" : "<extractor.*.user-agent>", "Accept" : "*/*", "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate" + "Accept-Encoding": "gzip, deflate", + "Referer" : "<extractor.*.referer>" } Description diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index a1a9d0d9..41cc0deb 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -23,9 +23,6 @@ class _500pxExtractor(Extractor): root = "https://500px.com" cookies_domain = ".500px.com" - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): data = self.metadata() diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 3ecae13d..fc16f43c 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -102,9 +102,6 @@ class _8chanBoardExtractor(_8chanExtractor): _8chanExtractor.__init__(self, match) _, self.board, self.page = match.groups() - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): page = text.parse_int(self.page, 1) url = "{}/{}/{}.json".format(self.root, self.board, page) diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index d6676946..b58b3d39 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -117,7 +117,6 @@ class ArtstationExtractor(Extractor): headers = { "Accept" : "application/json, text/plain, */*", "Origin" : self.root, - "Referer": self.root + "/", } if json: @@ -147,7 +146,6 @@ class ArtstationExtractor(Extractor): headers = { "Accept" : "*/*", "Origin" : self.root, - "Referer": self.root + "/", } return self.request( url, method="POST", headers=headers, json={}, diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index ac547a9e..fc5f9eff 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -35,9 +35,8 @@ class BehanceExtractor(Extractor): def _request_graphql(self, endpoint, variables): url = self.root + "/v3/graphql" headers = { - "Origin" : self.root, - "Referer": self.root + "/", - "X-BCP" : self._bcp, + "Origin": self.root, + "X-BCP" : self._bcp, "X-Requested-With": "XMLHttpRequest", } data = { diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 1318d0e4..dc48090a 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -42,7 +42,6 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): cdn = None files = [] append = files.append - headers = {"Referer": self.root + "/"} pos = page.index('class="grid-images') for url in text.extract_iter(page, '<a href="', '"', pos): @@ -63,7 +62,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): else: domain = domain.replace("cdn", "media-files", 1) url = urlunsplit((scheme, domain, path, query, fragment)) - append({"file": url, "_http_headers": headers}) + append({"file": url}) return files, { "album_id" : self.album_id, diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 399e3e24..72c7b7c6 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -310,6 +310,13 @@ class Extractor(): else: headers["Accept-Encoding"] = "gzip, deflate" + custom_referer = self.config("referer", True) + if custom_referer: + if isinstance(custom_referer, str): + headers["Referer"] = custom_referer + elif self.root: + headers["Referer"] = self.root + "/" + custom_headers = self.config("headers") if custom_headers: headers.update(custom_headers) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index b8e7164c..94c7f005 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1422,11 +1422,9 @@ class DeviantartEclipseAPI(): def _call(self, endpoint, params): url = "https://www.deviantart.com/_napi" + endpoint - headers = {"Referer": "https://www.deviantart.com/"} params["csrf_token"] = self.csrf_token or self._fetch_csrf_token() - response = self.request( - url, params=params, headers=headers, fatal=None) + response = self.request(url, params=params, fatal=None) if response.status_code == 404: raise exception.StopExtraction( diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index e9131136..44bfe7d8 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -45,7 +45,6 @@ class ExhentaiExtractor(Extractor): if self.version != "ex": self.cookies.set("nw", "1", domain=self.cookies_domain) - self.session.headers["Referer"] = self.root + "/" self.original = self.config("original", True) limits = self.config("limits", False) diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index b7463336..f1d51e21 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -22,7 +22,6 @@ class FantiaExtractor(Extractor): def _init(self): self.headers = { "Accept" : "application/json, text/plain, */*", - "Referer": self.root, "X-Requested-With": "XMLHttpRequest", } self._empty_plan = { @@ -65,11 +64,9 @@ class FantiaExtractor(Extractor): def _pagination(self, url): params = {"page": 1} - headers = self.headers.copy() - del headers["X-Requested-With"] while True: - page = self.request(url, params=params, headers=headers).text + page = self.request(url, params=params).text self._csrf_token(page) post_id = None diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index b8ec9be4..93ac5416 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -25,9 +25,6 @@ class FoolfuukaExtractor(BaseExtractor): if self.category == "b4k": self.remote = self._remote_direct - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): yield Message.Directory, self.metadata() for post in self.posts(): diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index a358019e..32ca1519 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -22,9 +22,6 @@ class HiperdexBase(): category = "hiperdex" root = "https://hiperdex.com" - def _init(self): - self.session.headers["Referer"] = self.root + "/" - @memcache(keyarg=1) def manga_data(self, manga, page=None): if not page: diff --git a/gallery_dl/extractor/hotleak.py b/gallery_dl/extractor/hotleak.py index 234f0db3..6d3184d9 100644 --- a/gallery_dl/extractor/hotleak.py +++ b/gallery_dl/extractor/hotleak.py @@ -21,9 +21,6 @@ class HotleakExtractor(Extractor): archive_fmt = "{type}_{creator}_{id}" root = "https://hotleak.vip" - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): for post in self.posts(): yield Message.Directory, post diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py index f9726f8e..aca101e1 100644 --- a/gallery_dl/extractor/imagefap.py +++ b/gallery_dl/extractor/imagefap.py @@ -23,9 +23,6 @@ class ImagefapExtractor(Extractor): archive_fmt = "{gallery_id}_{image_id}" request_interval = (2.0, 4.0) - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index 0955b4b1..8884d3ee 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -281,11 +281,7 @@ class ImgurAPI(): params["client_id"] = self.client_id params["page"] = 0 params["sort"] = "newest" - - headers = { - "Referer": "https://imgur.com/", - "Origin": "https://imgur.com", - } + headers = {"Origin": "https://imgur.com"} while True: data = self._call(endpoint, params, headers)["data"] diff --git a/gallery_dl/extractor/itaku.py b/gallery_dl/extractor/itaku.py index e9ec978b..1aef66e9 100644 --- a/gallery_dl/extractor/itaku.py +++ b/gallery_dl/extractor/itaku.py @@ -85,7 +85,6 @@ class ItakuAPI(): self.root = extractor.root + "/api" self.headers = { "Accept": "application/json, text/plain, */*", - "Referer": extractor.root + "/", } def galleries_images(self, username, section=None): diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 0c7d220c..894c6719 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -37,7 +37,6 @@ class KemonopartyExtractor(Extractor): Extractor.__init__(self, match) def _init(self): - self.session.headers["Referer"] = self.root + "/" self._prepare_ddosguard_cookies() self._find_inline = re.compile( r'src="(?:https?://(?:kemono|coomer)\.(?:party|su))?(/inline/[^"]+' diff --git a/gallery_dl/extractor/mangafox.py b/gallery_dl/extractor/mangafox.py index 2becfa54..d5907532 100644 --- a/gallery_dl/extractor/mangafox.py +++ b/gallery_dl/extractor/mangafox.py @@ -27,9 +27,6 @@ class MangafoxChapterExtractor(ChapterExtractor): self.urlbase = self.root + base ChapterExtractor.__init__(self, match, self.urlbase + "/1.html") - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def metadata(self, page): manga, pos = text.extract(page, "<title>", "") count, pos = text.extract( diff --git a/gallery_dl/extractor/mangakakalot.py b/gallery_dl/extractor/mangakakalot.py index 2b93392d..e1adf21e 100644 --- a/gallery_dl/extractor/mangakakalot.py +++ b/gallery_dl/extractor/mangakakalot.py @@ -31,9 +31,6 @@ class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): self.path = match.group(1) ChapterExtractor.__init__(self, match, self.root + self.path) - def _init(self): - self.session.headers['Referer'] = self.root + "/" - def metadata(self, page): _ , pos = text.extract(page, '', '<') manga , pos = text.extract(page, '', '<', pos) diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index d2175a02..46019ad8 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -23,8 +23,6 @@ class ManganeloBase(): super().__init__(match, "https://" + domain + path) def _init(self): - self.session.headers['Referer'] = self.root + "/" - if self._match_chapter is None: ManganeloBase._match_chapter = re.compile( r"(?:[Vv]ol\.?\s*(\d+)\s?)?" diff --git a/gallery_dl/extractor/naverwebtoon.py b/gallery_dl/extractor/naverwebtoon.py index c58056ec..72ee5b06 100644 --- a/gallery_dl/extractor/naverwebtoon.py +++ b/gallery_dl/extractor/naverwebtoon.py @@ -85,7 +85,6 @@ class NaverwebtoonComicExtractor(NaverwebtoonBase, Extractor): url = self.root + "/api/article/list" headers = { "Accept": "application/json, text/plain, */*", - "Referer": self.root + "/", } params = { "titleId": self.title_id, diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 55f5c285..1bcc915d 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -207,7 +207,6 @@ class NewgroundsExtractor(Extractor): headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", - "Referer": self.root, } sources = self.request(url, headers=headers).json()["sources"] @@ -478,7 +477,6 @@ class NewgroundsSearchExtractor(NewgroundsExtractor): headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", - "Referer": self.root, } params["inner"] = "1" params["page"] = 1 diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index 9dd9d7ab..b902404c 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -30,7 +30,6 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor): BaseExtractor.initialize(self) - self.session.headers["Referer"] = self.root + "/" self.user_name = None if self.category == "horne": self._extract_data = self._extract_data_horne diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index 9bff92ff..8c7ffe52 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -24,11 +24,11 @@ class NozomiExtractor(Extractor): filename_fmt = "{postid} {dataid}.{extension}" archive_fmt = "{dataid}" - def items(self): + def _init(self): + self.session.headers["Origin"] = self.root + def items(self): data = self.metadata() - self.session.headers["Origin"] = self.root - self.session.headers["Referer"] = self.root + "/" for post_id in map(str, self.posts()): url = "https://j.nozomi.la/post/{}/{}/{}.json".format( diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 0c5dfbe2..42957077 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -103,7 +103,6 @@ class PatreonExtractor(Extractor): def _pagination(self, url): headers = { - "Referer" : self.root + "/", "Content-Type": "application/vnd.api+json", } diff --git a/gallery_dl/extractor/pinterest.py b/gallery_dl/extractor/pinterest.py index 8afa397e..e9f124f1 100644 --- a/gallery_dl/extractor/pinterest.py +++ b/gallery_dl/extractor/pinterest.py @@ -325,7 +325,6 @@ class PinterestAPI(): "Accept" : "application/json, text/javascript, " "*/*, q=0.01", "Accept-Language" : "en-US,en;q=0.5", - "Referer" : self.root + "/", "X-Requested-With" : "XMLHttpRequest", "X-APP-VERSION" : "0c4af40", "X-CSRFToken" : csrf_token, diff --git a/gallery_dl/extractor/pornpics.py b/gallery_dl/extractor/pornpics.py index 1f14da07..4a6f0314 100644 --- a/gallery_dl/extractor/pornpics.py +++ b/gallery_dl/extractor/pornpics.py @@ -24,9 +24,6 @@ class PornpicsExtractor(Extractor): super().__init__(match) self.item = match.group(1) - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): for gallery in self.galleries(): gallery["_extractor"] = PornpicsGalleryExtractor diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index abe993a6..9a6c8a5a 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -34,7 +34,6 @@ class ReactorExtractor(BaseExtractor): self.category = netloc.rpartition(".")[0] def _init(self): - self.session.headers["Referer"] = self.root self.gif = self.config("gif", False) def items(self): diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index 52315106..57093091 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -193,7 +193,6 @@ class RedgifsAPI(): def __init__(self, extractor): self.extractor = extractor self.headers = { - "Referer" : extractor.root + "/", "authorization" : None, "content-type" : "application/json", "x-customheader": extractor.root + "/", diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index f9fe7fa3..745a351b 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -180,7 +180,6 @@ class SankakuAPI(): self.extractor = extractor self.headers = { "Accept" : "application/vnd.sankaku.api+json;v=2", - "Referer" : extractor.root + "/", "Platform": "web-app", "Origin" : extractor.root, } diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index f10ab622..55a0db00 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -45,7 +45,7 @@ class SkebExtractor(Extractor): """Return additional metadata""" def _pagination(self, url, params): - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} params["offset"] = 0 while True: @@ -69,7 +69,7 @@ class SkebExtractor(Extractor): def _get_post_data(self, user_name, post_num): url = "{}/api/users/{}/works/{}".format( self.root, user_name, post_num) - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} resp = self.request(url, headers=headers).json() creator = resp["creator"] post = { @@ -190,7 +190,6 @@ class SkebSearchExtractor(SkebExtractor): } headers = { "Origin": self.root, - "Referer": self.root + "/", "x-algolia-api-key": "9a4ce7d609e71bf29e977925e4c6740c", "x-algolia-application-id": "HB1JT3KRE9", } @@ -243,7 +242,7 @@ class SkebFollowingExtractor(SkebExtractor): url = "{}/api/users/{}/following_creators".format( self.root, self.user_name) params = {"sort": "date", "offset": 0, "limit": 90} - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} while True: data = self.request(url, params=params, headers=headers).json() diff --git a/gallery_dl/extractor/vipergirls.py b/gallery_dl/extractor/vipergirls.py index 456a0298..4ee252ef 100644 --- a/gallery_dl/extractor/vipergirls.py +++ b/gallery_dl/extractor/vipergirls.py @@ -26,9 +26,6 @@ class VipergirlsExtractor(Extractor): cookies_domain = ".vipergirls.to" cookies_names = ("vg_userid", "vg_password") - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): self.login() diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 51999455..168d5a0f 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -37,7 +37,6 @@ class WeiboExtractor(Extractor): cookies = _cookie_cache() if cookies is not None: self.cookies.update(cookies) - self.session.headers["Referer"] = self.root + "/" def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) From 0655ce1baedc3a453ecbfb00085562d5f8f9f011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 19 Sep 2023 00:15:26 +0200 Subject: [PATCH 52/53] [mangakakalot] update domain the old one still works, but it incurs a redirect --- gallery_dl/extractor/mangakakalot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/mangakakalot.py b/gallery_dl/extractor/mangakakalot.py index e1adf21e..0183b25f 100644 --- a/gallery_dl/extractor/mangakakalot.py +++ b/gallery_dl/extractor/mangakakalot.py @@ -19,7 +19,7 @@ BASE_PATTERN = r"(?:https?://)?(?:ww[\dw]?\.)?mangakakalot\.tv" class MangakakalotBase(): """Base class for mangakakalot extractors""" category = "mangakakalot" - root = "https://ww3.mangakakalot.tv" + root = "https://ww6.mangakakalot.tv" class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): From 7592c5e566253152c8a07f6ce8d517a9ba76cbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 19 Sep 2023 13:18:47 +0200 Subject: [PATCH 53/53] [patreon] fix extraction (#4547) --- gallery_dl/extractor/patreon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 42957077..729ceaf1 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -250,7 +250,7 @@ class PatreonExtractor(Extractor): def _extract_bootstrap(self, page): return util.json_loads(text.extr( - page, "window.patreon.bootstrap,", "\n});") + "}") + page, "window.patreon.bootstrap,", "});") + "}") class PatreonCreatorExtractor(PatreonExtractor):