diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36ab9534..168edb9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,8 +32,7 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - pip install "flake8<4" "importlib-metadata<5" - pip install youtube-dl + pip install flake8 youtube-dl - name: Install yt-dlp run: | @@ -53,7 +52,14 @@ jobs: - name: Lint with flake8 run: | - flake8 . + case "${{ matrix.python-version }}" in + 3.4|3.5|3.6|3.7) + flake8 --extend-exclude scripts/export_tests.py . + ;; + *) + flake8 . + ;; + esac - name: Run tests run: | diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c7f3712..ee1567a9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -390,7 +390,6 @@ Description * ``e621`` (*) * ``e926`` (*) * ``exhentai`` - * ``gfycat`` * ``idolcomplex`` * ``imgbb`` * ``inkbunny`` @@ -534,7 +533,7 @@ extractor.*.user-agent Type ``string`` Default - ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0"`` + ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0"`` Description User-Agent header value to be used for HTTP requests. @@ -566,6 +565,21 @@ Description browser would use HTTP/2. +extractor.*.referer +------------------- +Type + * ``bool`` + * ``string`` +Default + ``true`` +Description + Send `Referer `__ + headers with all outgoing HTTP requests. + + If this is a ``string``, send it as Referer + instead of the extractor's ``root`` domain. + + extractor.*.headers ------------------- Type @@ -577,7 +591,8 @@ Default "User-Agent" : "", "Accept" : "*/*", "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate" + "Accept-Encoding": "gzip, deflate", + "Referer" : "" } Description @@ -714,7 +729,7 @@ Type Default ``["oauth", "recursive", "test"]`` + current extractor category Example - ``["imgur", "gfycat:user", "*:image"]`` + ``["imgur", "redgifs:user", "*:image"]`` Description A list of extractor identifiers to ignore (or allow) when spawning child extractors for unknown URLs, @@ -723,7 +738,7 @@ Description Each identifier can be * A category or basecategory name (``"imgur"``, ``"mastodon"``) - * | A (base)category-subcategory pair, where both names are separated by a colon (``"gfycat:user"``). + * | A (base)category-subcategory pair, where both names are separated by a colon (``"redgifs:user"``). | Both names can be a `*` or left empty, matching all possible names (``"*:image"``, ``":user"``). Note: Any ``blacklist`` setting will automatically include @@ -1475,6 +1490,22 @@ Description * ``"exhentai.org"``: Use ``exhentai.org`` for all URLs +extractor.exhentai.fav +---------------------- +Type + ``string`` +Example + ``"4"`` +Description + After downloading a gallery, + add it to your account's favorites as the given category number. + + Note: Set this to `"favdel"` to remove galleries from your favorites. + + Note: This will remove any Favorite Notes when applied + to already favorited galleries. + + extractor.exhentai.limits ------------------------- Type @@ -1690,29 +1721,6 @@ Description even ones without a ``generic:`` prefix. -extractor.gfycat.format ------------------------ -Type - * ``string`` - * ``list`` of ``strings`` -Default - ``["mp4", "webm", "mobile", "gif"]`` -Description - List of names of the preferred animation format, which can be - ``"mp4"``, - ``"webm"``, - ``"mobile"``, - ``"gif"``, or - ``"webp"``. - - If a selected format is not available, the next one in the list will be - tried until an available format is found. - - If the format is given as ``string``, it will be extended with - ``["mp4", "webm", "mobile", "gif"]``. Use a list with one element to - restrict it to only one possible format. - - extractor.gofile.api-token -------------------------- Type @@ -3336,7 +3344,7 @@ extractor.twitter.users Type ``string`` Default - ``"timeline"`` + ``"user"`` Example ``"https://twitter.com/search?q=from:{legacy[screen_name]}"`` Description @@ -3347,7 +3355,8 @@ Description Special values: - * ``"timeline"``: ``https://twitter.com/i/user/{rest_id}`` + * ``"user"``: ``https://twitter.com/i/user/{rest_id}`` + * ``"timeline"``: ``https://twitter.com/id:{rest_id}/timeline`` * ``"tweets"``: ``https://twitter.com/id:{rest_id}/tweets`` * ``"media"``: ``https://twitter.com/id:{rest_id}/media`` diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf index da386dd6..c3f80493 100644 --- a/docs/gallery-dl-example.conf +++ b/docs/gallery-dl-example.conf @@ -165,7 +165,7 @@ "reddit": { "#": "only spawn child extractors for links to specific sites", - "whitelist": ["imgur", "redgifs", "gfycat"], + "whitelist": ["imgur", "redgifs"], "#": "put files from child extractors into the reddit directory", "parent-directory": true, diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index b5efc734..6345ded7 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -10,7 +10,7 @@ "proxy": null, "skip": true, - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", "retries": 4, "timeout": 30.0, "verify": true, @@ -125,10 +125,6 @@ "api-key": null, "user-id": null }, - "gfycat": - { - "format": ["mp4", "webm", "mobile", "gif"] - }, "gofile": { "api-token": null, "website-token": null @@ -339,7 +335,7 @@ "text-tweets": false, "twitpic": false, "unique": true, - "users": "timeline", + "users": "user", "videos": true }, "unsplash": diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 8a1aaa91..2e946f60 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -88,7 +88,7 @@ Consider all sites to be NSFW unless otherwise known. ArtStation https://www.artstation.com/ - Albums, Artwork Listings, Challenges, individual Images, Likes, Search Results, User Profiles + Albums, Artwork Listings, Challenges, Followed Users, individual Images, Likes, Search Results, User Profiles @@ -148,7 +148,7 @@ Consider all sites to be NSFW unless otherwise known. DeviantArt https://www.deviantart.com/ - Collections, Deviations, Favorites, Folders, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches + Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches OAuth @@ -226,7 +226,7 @@ Consider all sites to be NSFW unless otherwise known. Fur Affinity https://www.furaffinity.net/ - Favorites, Galleries, Posts, Scraps, Search Results, User Profiles + Favorites, Followed Users, Galleries, Posts, Scraps, Search Results, User Profiles Cookies @@ -247,12 +247,6 @@ Consider all sites to be NSFW unless otherwise known. Favorites, Pools, Posts, Redirects, Tag Searches - - Gfycat - https://gfycat.com/ - Collections, individual Images, Search Results, User Profiles - Supported - Gofile https://gofile.io/ @@ -357,7 +351,7 @@ Consider all sites to be NSFW unless otherwise known. Imagevenue - https://imagevenue.com/ + https://www.imagevenue.com/ individual Images @@ -406,13 +400,13 @@ Consider all sites to be NSFW unless otherwise known. Inkbunny https://inkbunny.net/ - Favorites, Pools, Posts, Search Results, User Profiles + Favorites, Followed Users, Pools, Posts, Search Results, User Profiles Supported Instagram https://www.instagram.com/ - Avatars, Collections, Guides, Highlights, Posts, Reels, Saved Posts, Stories, Tag Searches, Tagged Posts, User Profiles + Avatars, Collections, Followed Users, Guides, Highlights, Posts, Reels, Saved Posts, Stories, Tag Searches, Tagged Posts, User Profiles Cookies @@ -435,7 +429,7 @@ Consider all sites to be NSFW unless otherwise known. JPG Fish - https://jpeg.pet/ + https://jpg1.su/ Albums, individual Images, User Profiles @@ -574,7 +568,7 @@ Consider all sites to be NSFW unless otherwise known. Newgrounds https://www.newgrounds.com/ - Art, Audio, Favorites, Games, individual Images, Media Files, Movies, Search Results, User Profiles + Art, Audio, Favorites, Followed Users, Games, individual Images, Media Files, Movies, Search Results, User Profiles Supported @@ -699,7 +693,7 @@ Consider all sites to be NSFW unless otherwise known. Pornhub https://www.pornhub.com/ - Galleries, User Profiles + Galleries, Gifs, Photos, User Profiles @@ -783,7 +777,7 @@ Consider all sites to be NSFW unless otherwise known. Skeb https://skeb.jp/ - Posts, Search Results, User Profiles + Followed Users, Posts, Search Results, User Profiles @@ -879,7 +873,7 @@ Consider all sites to be NSFW unless otherwise known. Twitter https://twitter.com/ - Avatars, Backgrounds, Bookmarks, Events, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles + Avatars, Backgrounds, Bookmarks, Events, Followed Users, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles Supported @@ -1126,6 +1120,12 @@ Consider all sites to be NSFW unless otherwise known. Favorites, Pools, Posts, Tag Searches + + Xbooru + https://xbooru.com/ + Favorites, Pools, Posts, Tag Searches + + jschan Imageboards @@ -1165,19 +1165,19 @@ Consider all sites to be NSFW unless otherwise known. Misskey.io https://misskey.io/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles Lesbian.energy https://lesbian.energy/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles Sushi.ski https://sushi.ski/ - Favorites, Images from Notes, User Profiles + Favorites, Followed Users, Images from Notes, User Profiles @@ -1477,19 +1477,19 @@ Consider all sites to be NSFW unless otherwise known. mastodon.social https://mastodon.social/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth Pawoo https://pawoo.net/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth baraag https://baraag.net/ - Bookmarks, Images from Statuses, User Profiles + Bookmarks, Followed Users, Images from Statuses, User Profiles OAuth diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index ab72b8e1..d3a0f588 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -196,16 +196,15 @@ def main(): elif args.list_extractors: write = sys.stdout.write - fmt = "{}\n{}\nCategory: {} - Subcategory: {}{}\n\n".format + fmt = ("{}{}\nCategory: {} - Subcategory: {}" + "\nExample : {}\n\n").format for extr in extractor.extractors(): - if not extr.__doc__: - continue - test = next(extr._get_tests(), None) write(fmt( - extr.__name__, extr.__doc__, + extr.__name__, + "\n" + extr.__doc__ if extr.__doc__ else "", extr.category, extr.subcategory, - "\nExample : " + test[0] if test else "", + extr.example, )) elif args.clear_cache: @@ -297,7 +296,7 @@ def main(): return retval except KeyboardInterrupt: - sys.exit("\nKeyboardInterrupt") + raise SystemExit("\nKeyboardInterrupt") except BrokenPipeError: pass except OSError as exc: diff --git a/gallery_dl/__main__.py b/gallery_dl/__main__.py index 637d463f..441009fe 100644 --- a/gallery_dl/__main__.py +++ b/gallery_dl/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -17,4 +17,4 @@ if __package__ is None and not hasattr(sys, "frozen"): import gallery_dl if __name__ == "__main__": - sys.exit(gallery_dl.main()) + raise SystemExit(gallery_dl.main()) diff --git a/gallery_dl/actions.py b/gallery_dl/actions.py index 15ca31ec..883e38b0 100644 --- a/gallery_dl/actions.py +++ b/gallery_dl/actions.py @@ -9,7 +9,6 @@ """ """ import re -import sys import logging import operator from . import util, exception @@ -98,7 +97,7 @@ def action_exit(opts): pass def _exit(args): - sys.exit(opts) + raise SystemExit(opts) return _exit diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 767185fb..4be6c532 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -100,12 +100,12 @@ def load(files=None, strict=False, loads=util.json_loads): except OSError as exc: if strict: log.error(exc) - sys.exit(1) + raise SystemExit(1) except Exception as exc: log.error("%s when loading '%s': %s", exc.__class__.__name__, path, exc) if strict: - sys.exit(2) + raise SystemExit(2) else: if not _config: _config.update(conf) diff --git a/gallery_dl/extractor/2chan.py b/gallery_dl/extractor/2chan.py index 92ea6caf..337ba480 100644 --- a/gallery_dl/extractor/2chan.py +++ b/gallery_dl/extractor/2chan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,26 +20,8 @@ class _2chanThreadExtractor(Extractor): filename_fmt = "{tim}.{extension}" archive_fmt = "{board}_{thread}_{tim}" url_fmt = "https://{server}.2chan.net/{board}/src/{filename}" - pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/]+)/res/(\d+)" - test = ("https://dec.2chan.net/70/res/14565.htm", { - "pattern": r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", - "count": ">= 3", - "keyword": { - "board": "70", - "board_name": "新板提案", - "com": str, - "fsize": r"re:\d+", - "name": "名無し", - "no": r"re:1[45]\d\d\d", - "now": r"re:22/../..\(.\)..:..:..", - "post": "無題", - "server": "dec", - "thread": "14565", - "tim": r"re:^\d{13}$", - "time": r"re:^\d{10}$", - "title": "ﾋﾛｱｶ板" - }, - }) + pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/?#]+)/res/(\d+)" + example = "https://dec.2chan.net/12/res/12345.htm" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index f1426900..0c978897 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -21,26 +21,7 @@ class _2chenThreadExtractor(Extractor): filename_fmt = "{time} {filename}.{extension}" archive_fmt = "{board}_{thread}_{hash}_{time}" pattern = BASE_PATTERN + r"/([^/?#]+)/(\d+)" - test = ( - ("https://sturdychan.help/tv/268929", { - "pattern": r"https://sturdychan\.help/assets/images" - r"/src/\w{40}\.\w+$", - "count": ">= 179", - "keyword": { - "board": "tv", - "date": "type:datetime", - "hash": r"re:[0-9a-f]{40}", - "name": "Anonymous", - "no": r"re:\d+", - "thread": "268929", - "time": int, - "title": "「/ttg/ #118: 🇧🇷 edition」", - "url": str, - }, - }), - ("https://2chen.club/tv/1"), - ("https://2chen.moe/jp/303786"), - ) + example = "https://sturdychan.help/a/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -101,14 +82,7 @@ class _2chenBoardExtractor(Extractor): subcategory = "board" root = "https://sturdychan.help" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/catalog|/?$)" - test = ( - ("https://sturdychan.help/co/", { - "pattern": _2chenThreadExtractor.pattern - }), - ("https://2chen.moe/co"), - ("https://2chen.club/tv"), - ("https://2chen.moe/co/catalog"), - ) + example = "https://sturdychan.help/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/35photo.py b/gallery_dl/extractor/35photo.py index f86691d4..773116ea 100644 --- a/gallery_dl/extractor/35photo.py +++ b/gallery_dl/extractor/35photo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -101,20 +101,7 @@ class _35photoUserExtractor(_35photoExtractor): subcategory = "user" pattern = (r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro" r"/(?!photo_|genre_|tags/|rating/)([^/?#]+)") - test = ( - ("https://35photo.pro/liya", { - "pattern": r"https://([a-z][0-9]\.)?35photo\.pro" - r"/photos_(main|series)/.*\.jpg", - "count": 9, - }), - ("https://35photo.pro/suhoveev", { - # last photo ID (1267028) isn't given as 'photo-id="" - # there are only 23 photos without the last one - "count": ">= 33", - }), - ("https://en.35photo.pro/liya"), - ("https://ru.35photo.pro/liya"), - ) + example = "https://35photo.pro/USER" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -143,11 +130,7 @@ class _35photoTagExtractor(_35photoExtractor): directory_fmt = ("{category}", "Tags", "{search_tag}") archive_fmt = "t{search_tag}_{id}_{num}" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/tags/([^/?#]+)" - test = ("https://35photo.pro/tags/landscape/", { - "range": "1-25", - "count": 25, - "archive": False, - }) + example = "https://35photo.pro/tags/TAG/" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -180,7 +163,7 @@ class _35photoGenreExtractor(_35photoExtractor): directory_fmt = ("{category}", "Genre", "{genre}") archive_fmt = "g{genre_id}_{id}_{num}" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/genre_(\d+)(/new/)?" - test = ("https://35photo.pro/genre_109/",) + example = "https://35photo.pro/genre_12345/" def __init__(self, match): _35photoExtractor.__init__(self, match) @@ -212,24 +195,7 @@ class _35photoImageExtractor(_35photoExtractor): """Extractor for individual images from 35photo.pro""" subcategory = "image" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/photo_(\d+)" - test = ("https://35photo.pro/photo_753340/", { - "count": 1, - "keyword": { - "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", - "id" : 753340, - "title" : "Winter walk", - "description": str, - "tags" : list, - "views" : int, - "favorites" : int, - "score" : int, - "type" : 0, - "date" : "15 авг, 2014", - "user" : "liya", - "user_id" : 20415, - "user_name" : "Liya Mirzaeva", - }, - }) + example = "https://35photo.pro/photo_12345/" def __init__(self, match): _35photoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/3dbooru.py b/gallery_dl/extractor/3dbooru.py index e83bca75..c1bc9e5e 100644 --- a/gallery_dl/extractor/3dbooru.py +++ b/gallery_dl/extractor/3dbooru.py @@ -27,10 +27,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor): """Extractor for images from behoimi.org based on search-tags""" pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org/post" r"(?:/(?:index)?)?\?tags=(?P[^&#]+)") - test = ("http://behoimi.org/post?tags=himekawa_azuru+dress", { - "url": "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", - "content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", - }) + example = "http://behoimi.org/post?tags=TAG" def posts(self): params = {"tags": self.tags} @@ -40,10 +37,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor): class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): """Extractor for image-pools from behoimi.org""" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P\d+)" - test = ("http://behoimi.org/pool/show/27", { - "url": "da75d2d1475449d5ef0c266cb612683b110a30f2", - "content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", - }) + example = "http://behoimi.org/pool/show/12345" def posts(self): params = {"tags": "pool:" + self.pool_id} @@ -53,17 +47,7 @@ class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor): """Extractor for single images from behoimi.org""" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P\d+)" - test = ("http://behoimi.org/post/show/140852", { - "url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", - "content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", - "options": (("tags", True),), - "keyword": { - "tags_character": "furude_rika", - "tags_copyright": "higurashi_no_naku_koro_ni", - "tags_model": "himekawa_azuru", - "tags_general": str, - }, - }) + example = "http://behoimi.org/post/show/12345" def posts(self): params = {"tags": "id:" + self.post_id} @@ -76,7 +60,4 @@ class _3dbooruPopularExtractor( pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org" r"/post/popular_(?Pby_(?:day|week|month)|recent)" r"(?:\?(?P[^#]*))?") - test = ("http://behoimi.org/post/popular_by_month?month=2&year=2013", { - "pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", - "count": 20, - }) + example = "http://behoimi.org/post/popular_by_month" diff --git a/gallery_dl/extractor/4chan.py b/gallery_dl/extractor/4chan.py index bf9615d0..2db60422 100644 --- a/gallery_dl/extractor/4chan.py +++ b/gallery_dl/extractor/4chan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2019 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,17 +21,7 @@ class _4chanThreadExtractor(Extractor): archive_fmt = "{board}_{thread}_{tim}" pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org" r"/([^/]+)/thread/(\d+)") - test = ( - ("https://boards.4chan.org/tg/thread/15396072/", { - "url": "39082ad166161966d7ba8e37f2173a824eb540f0", - "keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - "content": "20b7b51afa51c9c31a0020a0737b889532c8d7ec", - }), - ("https://boards.4channel.org/tg/thread/15396072/", { - "url": "39082ad166161966d7ba8e37f2173a824eb540f0", - "keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - }), - ) + example = "https://boards.4channel.org/a/thread/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -65,10 +55,7 @@ class _4chanBoardExtractor(Extractor): category = "4chan" subcategory = "board" pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$" - test = ("https://boards.4channel.org/po/", { - "pattern": _4chanThreadExtractor.pattern, - "count": ">= 100", - }) + example = "https://boards.4channel.org/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/4chanarchives.py b/gallery_dl/extractor/4chanarchives.py index 041e6a35..f018d3ec 100644 --- a/gallery_dl/extractor/4chanarchives.py +++ b/gallery_dl/extractor/4chanarchives.py @@ -21,21 +21,7 @@ class _4chanarchivesThreadExtractor(Extractor): filename_fmt = "{no}-{filename}.{extension}" archive_fmt = "{board}_{thread}_{no}" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)/thread/(\d+)" - test = ( - ("https://4chanarchives.com/board/c/thread/2707110", { - "pattern": r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", - "count": 2, - "keyword": { - "board": "c", - "com": str, - "name": "Anonymous", - "no": int, - "thread": "2707110", - "time": r"re:2016-07-1\d \d\d:\d\d:\d\d", - "title": "Ren Kagami from 'Oyako Neburi'", - }, - }), - ) + example = "https://4chanarchives.com/board/a/thread/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -106,15 +92,7 @@ class _4chanarchivesBoardExtractor(Extractor): subcategory = "board" root = "https://4chanarchives.com" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)(?:/(\d+))?/?$" - test = ( - ("https://4chanarchives.com/board/c/", { - "pattern": _4chanarchivesThreadExtractor.pattern, - "range": "1-40", - "count": 40, - }), - ("https://4chanarchives.com/board/c"), - ("https://4chanarchives.com/board/c/10"), - ) + example = "https://4chanarchives.com/board/a/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index 4d1307e1..41cc0deb 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -23,9 +23,6 @@ class _500pxExtractor(Extractor): root = "https://500px.com" cookies_domain = ".500px.com" - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): data = self.metadata() @@ -96,15 +93,7 @@ class _500pxUserExtractor(_500pxExtractor): """Extractor for photos from a user's photostream on 500px.com""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!photo/|liked)(?:p/)?([^/?#]+)/?(?:$|[?#])" - test = ( - ("https://500px.com/p/light_expression_photography", { - "pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", - "range": "1-99", - "count": 99, - }), - ("https://500px.com/light_expression_photography"), - ("https://web.500px.com/light_expression_photography"), - ) + example = "https://500px.com/USER" def __init__(self, match): _500pxExtractor.__init__(self, match) @@ -134,17 +123,7 @@ class _500pxGalleryExtractor(_500pxExtractor): directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}") pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?" r"([^/?#]+)/galleries/([^/?#]+)") - test = ( - ("https://500px.com/p/fashvamp/galleries/lera", { - "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", - "count": 3, - "keyword": { - "gallery": dict, - "user": dict, - }, - }), - ("https://500px.com/fashvamp/galleries/lera"), - ) + example = "https://500px.com/USER/galleries/GALLERY" def __init__(self, match): _500pxExtractor.__init__(self, match) @@ -200,7 +179,7 @@ class _500pxFavoriteExtractor(_500pxExtractor): """Extractor for favorite 500px photos""" subcategory = "favorite" pattern = BASE_PATTERN + r"/liked/?$" - test = ("https://500px.com/liked",) + example = "https://500px.com/liked" def photos(self): variables = {"pageSize": 20} @@ -224,50 +203,7 @@ class _500pxImageExtractor(_500pxExtractor): """Extractor for individual images from 500px.com""" subcategory = "image" pattern = BASE_PATTERN + r"/photo/(\d+)" - test = ("https://500px.com/photo/222049255/queen-of-coasts", { - "url": "fbdf7df39325cae02f5688e9f92935b0e7113315", - "count": 1, - "keyword": { - "camera": "Canon EOS 600D", - "camera_info": dict, - "comments": list, - "comments_count": int, - "created_at": "2017-08-01T08:40:05+00:00", - "description": str, - "editored_by": None, - "editors_choice": False, - "extension": "jpg", - "feature": "popular", - "feature_date": "2017-08-01T09:58:28+00:00", - "focal_length": "208", - "height": 3111, - "id": 222049255, - "image_format": "jpg", - "image_url": list, - "images": list, - "iso": "100", - "lens": "EF-S55-250mm f/4-5.6 IS II", - "lens_info": dict, - "liked": None, - "location": None, - "location_details": dict, - "name": "Queen Of Coasts", - "nsfw": False, - "privacy": False, - "profile": True, - "rating": float, - "status": 1, - "tags": list, - "taken_at": "2017-05-04T17:36:51+00:00", - "times_viewed": int, - "url": "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", - "user": dict, - "user_id": 12847235, - "votes_count": int, - "watermark": True, - "width": 4637, - }, - }) + example = "https://500px.com/photo/12345/TITLE" def __init__(self, match): _500pxExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index 2d043868..fc16f43c 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -57,48 +57,7 @@ class _8chanThreadExtractor(_8chanExtractor): filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}" archive_fmt = "{boardUri}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://8chan.moe/vhs/res/4.html", { - "pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", - "count": 14, - "keyword": { - "archived": False, - "autoSage": False, - "boardDescription": "Film and Cinema", - "boardMarkdown": None, - "boardName": "Movies", - "boardUri": "vhs", - "creation": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", - "cyclic": False, - "email": None, - "id": "re:^[0-9a-f]{6}$", - "locked": False, - "markdown": str, - "maxFileCount": 5, - "maxFileSize": "32.00 MB", - "maxMessageLength": 8001, - "message": str, - "mime": str, - "name": "Anonymous", - "num": int, - "originalName": str, - "path": r"re:/.media/[0-9a-f]{64}\.\w+$", - "pinned": False, - "postId": int, - "signedRole": None, - "size": int, - "threadId": 4, - "thumb": r"re:/.media/t_[0-9a-f]{64}$", - "uniquePosters": 9, - "usesCustomCss": True, - "usesCustomJs": False, - "?wsPort": 8880, - "?wssPort": 2087, - }, - }), - ("https://8chan.se/vhs/res/4.html"), - ("https://8chan.cc/vhs/res/4.html"), - ) + example = "https://8chan.moe/a/res/12345.html" def __init__(self, match): _8chanExtractor.__init__(self, match) @@ -137,23 +96,12 @@ class _8chanBoardExtractor(_8chanExtractor): """Extractor for 8chan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$" - test = ( - ("https://8chan.moe/vhs/"), - ("https://8chan.moe/vhs/2.html", { - "pattern": _8chanThreadExtractor.pattern, - "count": 23, - }), - ("https://8chan.se/vhs/"), - ("https://8chan.cc/vhs/"), - ) + example = "https://8chan.moe/a/" def __init__(self, match): _8chanExtractor.__init__(self, match) _, self.board, self.page = match.groups() - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): page = text.parse_int(self.page, 1) url = "{}/{}/{}.json".format(self.root, self.board, page) diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index 584c6d2e..f88a0c67 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -22,51 +22,7 @@ class _8musesAlbumExtractor(Extractor): root = "https://comics.8muses.com" pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" r"(/comics/album/[^?#]+)(\?[^#]+)?") - test = ( - ("https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", { - "url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", - "pattern": r"https://comics.8muses.com/image/fl/[\w-]+", - "keyword": { - "url" : str, - "hash" : str, - "page" : int, - "count": 6, - "album": { - "id" : 10467, - "title" : "Liar", - "path" : "Fakku Comics/mogg/Liar", - "parts" : ["Fakku Comics", "mogg", "Liar"], - "private": False, - "url" : "https://comics.8muses.com/comics" - "/album/Fakku-Comics/mogg/Liar", - "parent" : 10464, - "views" : int, - "likes" : int, - "date" : "dt:2018-07-10 00:00:00", - }, - }, - }), - ("https://www.8muses.com/comics/album/Fakku-Comics/santa", { - "count": ">= 3", - "pattern": pattern, - "keyword": { - "url" : str, - "name" : str, - "private": False, - }, - }), - # custom sorting - ("https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", { - "count": ">= 70", - "keyword": {"name": r"re:^[R-Zr-z]"}, - }), - # non-ASCII characters - (("https://comics.8muses.com/comics/album/Various-Authors/Chessire88" - "/From-Trainers-to-Pokmons"), { - "count": 2, - "keyword": {"name": "re:From Trainers to Pokémons"}, - }), - ) + example = "https://comics.8muses.com/comics/album/PATH/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 443bec37..44c08a29 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -50,7 +50,6 @@ modules = [ "gelbooru", "gelbooru_v01", "gelbooru_v02", - "gfycat", "gofile", "hbrowse", "hentai2read", diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 8160e480..1617414a 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,17 +18,7 @@ class AdultempireGalleryExtractor(GalleryExtractor): root = "https://www.adultempire.com" pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" r"(/(\d+)/gallery\.html)") - test = ( - ("https://www.adultempire.com/5998/gallery.html", { - "range": "1", - "keyword": "5b3266e69801db0d78c22181da23bc102886e027", - "content": "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", - }), - ("https://www.adultdvdempire.com/5683/gallery.html", { - "url": "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", - "keyword": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", - }), - ) + example = "https://www.adultempire.com/12345/gallery.html" def __init__(self, match): GalleryExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py index dbc197ee..8064e789 100644 --- a/gallery_dl/extractor/architizer.py +++ b/gallery_dl/extractor/architizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,25 +21,7 @@ class ArchitizerProjectExtractor(GalleryExtractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{gid}_{num}" pattern = r"(?:https?://)?architizer\.com/projects/([^/?#]+)" - test = ("https://architizer.com/projects/house-lo/", { - "pattern": r"https://architizer-prod\.imgix\.net/media/mediadata" - r"/uploads/.+\.jpg$", - "keyword": { - "count": 27, - "description": str, - "firm": "Atelier Lina Bellovicova", - "gid": "225496", - "location": "Czechia", - "num": int, - "size": "1000 sqft - 3000 sqft", - "slug": "house-lo", - "status": "Built", - "subcategory": "project", - "title": "House LO", - "type": "Residential › Private House", - "year": "2020", - }, - }) + example = "https://architizer.com/projects/NAME/" def __init__(self, match): url = "{}/projects/{}/".format(self.root, match.group(1)) @@ -47,11 +29,13 @@ class ArchitizerProjectExtractor(GalleryExtractor): def metadata(self, page): extr = text.extract_from(page) + extr('id="Pages"', "") + return { - "title" : extr("data-name='", "'"), - "slug" : extr("data-slug='", "'"), - "gid" : extr("data-gid='", "'").rpartition(".")[2], - "firm" : extr("data-firm-leaders-str='", "'"), + "title" : extr('data-name="', '"'), + "slug" : extr('data-slug="', '"'), + "gid" : extr('data-gid="', '"').rpartition(".")[2], + "firm" : extr('data-firm-leaders-str="', '"'), "location" : extr("

", "<").strip(), "type" : text.unescape(text.remove_html(extr( '
Type
', '= 90", - }) + example = "https://architizer.com/firms/NAME/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index 77d5fbd2..b58b3d39 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -117,7 +117,6 @@ class ArtstationExtractor(Extractor): headers = { "Accept" : "application/json, text/plain, */*", "Origin" : self.root, - "Referer": self.root + "/", } if json: @@ -147,7 +146,6 @@ class ArtstationExtractor(Extractor): headers = { "Accept" : "*/*", "Origin" : self.root, - "Referer": self.root + "/", } return self.request( url, method="POST", headers=headers, json={}, @@ -178,17 +176,7 @@ class ArtstationUserExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)(?:/albums/all)?" r"|((?!www)\w+)\.artstation\.com(?:/projects)?)/?$") - test = ( - ("https://www.artstation.com/sungchoi/", { - "pattern": r"https://\w+\.artstation\.com/p/assets/images" - r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "range": "1-10", - "count": ">= 10", - }), - ("https://www.artstation.com/sungchoi/albums/all/"), - ("https://sungchoi.artstation.com/"), - ("https://sungchoi.artstation.com/projects/"), - ) + example = "https://www.artstation.com/USER" def projects(self): url = "{}/users/{}/projects.json".format(self.root, self.user) @@ -205,15 +193,7 @@ class ArtstationAlbumExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)" r"|((?!www)\w+)\.artstation\.com)/albums/(\d+)") - test = ( - ("https://www.artstation.com/huimeiye/albums/770899", { - "count": 2, - }), - ("https://www.artstation.com/huimeiye/albums/770898", { - "exception": exception.NotFoundError, - }), - ("https://huimeiye.artstation.com/albums/770899"), - ) + example = "https://www.artstation.com/USER/albums/12345" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -247,17 +227,7 @@ class ArtstationLikesExtractor(ArtstationExtractor): archive_fmt = "f_{userinfo[id]}_{asset[id]}" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)/likes/?") - test = ( - ("https://www.artstation.com/mikf/likes", { - "pattern": r"https://\w+\.artstation\.com/p/assets/images" - r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "count": 6, - }), - # no likes - ("https://www.artstation.com/sungchoi/likes", { - "count": 0, - }), - ) + example = "https://www.artstation.com/USER/likes" def projects(self): url = "{}/users/{}/likes.json".format(self.root, self.user) @@ -274,14 +244,7 @@ class ArtstationChallengeExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/contests/[^/?#]+/challenges/(\d+)" r"/?(?:\?sorting=([a-z]+))?") - test = ( - ("https://www.artstation.com/contests/thu-2017/challenges/20"), - (("https://www.artstation.com/contests/beyond-human" - "/challenges/23?sorting=winners"), { - "range": "1-30", - "count": 30, - }), - ) + example = "https://www.artstation.com/contests/NAME/challenges/12345" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -327,10 +290,7 @@ class ArtstationSearchExtractor(ArtstationExtractor): archive_fmt = "s_{search[query]}_{asset[id]}" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" r"/search/?\?([^#]+)") - test = ("https://www.artstation.com/search?query=ancient&sort_by=rank", { - "range": "1-20", - "count": 20, - }) + example = "https://www.artstation.com/search?query=QUERY" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -377,10 +337,7 @@ class ArtstationArtworkExtractor(ArtstationExtractor): archive_fmt = "A_{asset[id]}" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" r"/artwork/?\?([^#]+)") - test = ("https://www.artstation.com/artwork?sorting=latest", { - "range": "1-20", - "count": 20, - }) + example = "https://www.artstation.com/artwork?sorting=SORT" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -400,32 +357,7 @@ class ArtstationImageExtractor(ArtstationExtractor): pattern = (r"(?:https?://)?(?:" r"(?:\w+\.)?artstation\.com/(?:artwork|projects|search)" r"|artstn\.co/p)/(\w+)") - test = ( - ("https://www.artstation.com/artwork/LQVJr", { - "pattern": r"https?://\w+\.artstation\.com/p/assets" - r"/images/images/008/760/279/4k/.+", - "content": "7b113871465fdc09d127adfdc2767d51cf45a7e9", - # SHA1 hash without _no_cache() - # "content": "44b80f9af36d40efc5a2668cdd11d36d6793bae9", - }), - # multiple images per project - ("https://www.artstation.com/artwork/Db3dy", { - "count": 4, - }), - # embedded youtube video - ("https://www.artstation.com/artwork/g4WPK", { - "range": "2", - "options": (("external", True),), - "pattern": "ytdl:https://www.youtube.com/embed/JNFfJtwwrU0", - }), - # 404 (#3016) - ("https://www.artstation.com/artwork/3q3mXB", { - "count": 0, - }), - # alternate URL patterns - ("https://sungchoi.artstation.com/projects/LQVJr"), - ("https://artstn.co/p/LQVJr"), - ) + example = "https://www.artstation.com/artwork/abcde" def __init__(self, match): ArtstationExtractor.__init__(self, match) @@ -453,10 +385,7 @@ class ArtstationFollowingExtractor(ArtstationExtractor): subcategory = "following" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" r"/(?!artwork|projects|search)([^/?#]+)/following") - test = ("https://www.artstation.com/sungchoi/following", { - "pattern": ArtstationUserExtractor.pattern, - "count": ">= 50", - }) + example = "https://www.artstation.com/USER/following" def items(self): url = "{}/users/{}/following.json".format(self.root, self.user) diff --git a/gallery_dl/extractor/aryion.py b/gallery_dl/extractor/aryion.py index 89a8319a..576bc832 100644 --- a/gallery_dl/extractor/aryion.py +++ b/gallery_dl/extractor/aryion.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -176,16 +176,7 @@ class AryionGalleryExtractor(AryionExtractor): subcategory = "gallery" categorytransfer = True pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)" - test = ( - ("https://aryion.com/g4/gallery/jameshoward", { - "options": (("recursive", False),), - "pattern": r"https://aryion\.com/g4/data\.php\?id=\d+$", - "range": "48-52", - "count": 5, - }), - ("https://aryion.com/g4/user/jameshoward"), - ("https://aryion.com/g4/latest.php?name=jameshoward"), - ) + example = "https://aryion.com/g4/gallery/USER" def __init__(self, match): AryionExtractor.__init__(self, match) @@ -215,9 +206,7 @@ class AryionTagExtractor(AryionExtractor): directory_fmt = ("{category}", "tags", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/tags\.php\?([^#]+)" - test = ("https://aryion.com/g4/tags.php?tag=star+wars&p=19", { - "count": ">= 5", - }) + example = "https://aryion.com/g4/tags.php?tag=TAG" def _init(self): self.params = text.parse_query(self.user) @@ -235,40 +224,7 @@ class AryionPostExtractor(AryionExtractor): """Extractor for individual posts on eka's portal""" subcategory = "post" pattern = BASE_PATTERN + r"/view/(\d+)" - test = ( - ("https://aryion.com/g4/view/510079", { - "url": "f233286fa5558c07ae500f7f2d5cb0799881450e", - "keyword": { - "artist" : "jameshoward", - "user" : "jameshoward", - "filename" : "jameshoward-510079-subscribestar_150", - "extension": "jpg", - "id" : 510079, - "width" : 1665, - "height" : 1619, - "size" : 784239, - "title" : "I'm on subscribestar now too!", - "description": r"re:Doesn't hurt to have a backup, right\?", - "tags" : ["Non-Vore", "subscribestar"], - "date" : "dt:2019-02-16 19:30:34", - "path" : [], - "views" : int, - "favorites": int, - "comments" : int, - "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", - }, - }), - # x-folder (#694) - ("https://aryion.com/g4/view/588928", { - "pattern": pattern, - "count": ">= 8", - }), - # x-comic-folder (#945) - ("https://aryion.com/g4/view/537379", { - "pattern": pattern, - "count": 2, - }), - ) + example = "https://aryion.com/g4/view/12345" def posts(self): post_id, self.user = self.user, None diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py index 638fedc2..54aaac4e 100644 --- a/gallery_dl/extractor/bbc.py +++ b/gallery_dl/extractor/bbc.py @@ -23,18 +23,7 @@ class BbcGalleryExtractor(GalleryExtractor): filename_fmt = "{num:>02}.{extension}" archive_fmt = "{programme}_{num}" pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$" - test = ( - ("https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", { - "pattern": r"https://ichef\.bbci\.co\.uk" - r"/images/ic/1920xn/\w+\.jpg", - "count": 37, - "keyword": { - "programme": "p084qtzs", - "path": ["BBC One", "Doctor Who", "The Timeless Children"], - }, - }), - ("https://www.bbc.co.uk/programmes/p084qtzs"), - ) + example = "https://www.bbc.co.uk/programmes/PATH" def metadata(self, page): data = util.json_loads(text.extr( @@ -72,17 +61,7 @@ class BbcProgrammeExtractor(Extractor): subcategory = "programme" root = "https://www.bbc.co.uk" pattern = BASE_PATTERN + r"[^/?#]+/galleries)(?:/?\?page=(\d+))?" - test = ( - ("https://www.bbc.co.uk/programmes/b006q2x0/galleries", { - "pattern": BbcGalleryExtractor.pattern, - "range": "1-50", - "count": ">= 50", - }), - ("https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", { - "pattern": BbcGalleryExtractor.pattern, - "count": ">= 100", - }), - ) + example = "https://www.bbc.co.uk/programmes/ID/galleries" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index cf123d91..fc5f9eff 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -35,9 +35,8 @@ class BehanceExtractor(Extractor): def _request_graphql(self, endpoint, variables): url = self.root + "/v3/graphql" headers = { - "Origin" : self.root, - "Referer": self.root + "/", - "X-BCP" : self._bcp, + "Origin": self.root, + "X-BCP" : self._bcp, "X-Requested-With": "XMLHttpRequest", } data = { @@ -84,43 +83,7 @@ class BehanceGalleryExtractor(BehanceExtractor): filename_fmt = "{category}_{id}_{num:>02}.{extension}" archive_fmt = "{id}_{num}" pattern = r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)" - test = ( - ("https://www.behance.net/gallery/17386197/A-Short-Story", { - "count": 2, - "url": "ab79bd3bef8d3ae48e6ac74fd995c1dfaec1b7d2", - "keyword": { - "id": 17386197, - "name": 're:"Hi". A short story about the important things ', - "owners": ["Place Studio", "Julio César Velazquez"], - "fields": ["Animation", "Character Design", "Directing"], - "tags": list, - "module": dict, - "date": "dt:2014-06-03 15:41:51", - }, - }), - ("https://www.behance.net/gallery/21324767/Nevada-City", { - "count": 6, - "url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", - "keyword": {"owners": ["Alex Strohl"]}, - }), - # 'media_collection' modules - ("https://www.behance.net/gallery/88276087/Audi-R8-RWD", { - "count": 20, - "url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", - "pattern": r"https://mir-s3-cdn-cf\.behance\.net/project_modules" - r"/source/[0-9a-f]+.[0-9a-f]+\.jpg" - }), - # 'video' modules (#1282) - ("https://www.behance.net/gallery/101185577/COLCCI", { - "pattern": r"https://cdn-prod-ccv\.adobe\.com/\w+" - r"/rend/\w+_720\.mp4\?", - "count": 3, - }), - # mature content (#4417) - ("https://www.behance.net/gallery/177464639/Kimori", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://www.behance.net/gallery/12345/TITLE" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -177,7 +140,13 @@ class BehanceGalleryExtractor(BehanceExtractor): append((url, module)) elif mtype == "VideoModule": - renditions = module["videoData"]["renditions"] + try: + renditions = module["videoData"]["renditions"] + except Exception: + self.log.warning("No download URLs for video %s", + module.get("id") or "???") + continue + try: url = [ r["url"] for r in renditions @@ -186,6 +155,7 @@ class BehanceGalleryExtractor(BehanceExtractor): except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) url = "ytdl:" + renditions[-1]["url"] + append((url, module)) elif mtype == "MediaCollectionModule": @@ -210,10 +180,7 @@ class BehanceUserExtractor(BehanceExtractor): subcategory = "user" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$" - test = ("https://www.behance.net/alexstrohl", { - "count": ">= 11", - "pattern": BehanceGalleryExtractor.pattern, - }) + example = "https://www.behance.net/USER" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -223,7 +190,7 @@ class BehanceUserExtractor(BehanceExtractor): endpoint = "GetProfileProjects" variables = { "username": self.user, - "after" : "MAo=", + "after" : "MAo=", # "0" in base64 } while True: @@ -241,10 +208,7 @@ class BehanceCollectionExtractor(BehanceExtractor): subcategory = "collection" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)" - test = ("https://www.behance.net/collection/71340149/inspiration", { - "count": ">= 150", - "pattern": BehanceGalleryExtractor.pattern, - }) + example = "https://www.behance.net/collection/12345/TITLE" def __init__(self, match): BehanceExtractor.__init__(self, match) @@ -253,7 +217,7 @@ class BehanceCollectionExtractor(BehanceExtractor): def galleries(self): endpoint = "GetMoodboardItemsAndRecommendations" variables = { - "afterItem": "MAo=", + "afterItem": "MAo=", # "0" in base64 "firstItem": 40, "id" : int(self.collection_id), "shouldGetItems" : True, diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index b25af8f9..d75c3498 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -95,59 +95,8 @@ class BloggerExtractor(Extractor): class BloggerPostExtractor(BloggerExtractor): """Extractor for a single blog post""" subcategory = "post" - pattern = BASE_PATTERN + r"(/\d{4}/\d\d/[^/?#]+\.html)" - test = ( - ("https://julianbphotography.blogspot.com/2010/12/moon-rise.html", { - "url": "9928429fb62f712eb4de80f53625eccecc614aae", - "pattern": r"https://3.bp.blogspot.com/.*/s0/Icy-Moonrise-.*.jpg", - "keyword": { - "blog": { - "date" : "dt:2010-11-21 18:19:42", - "description": "", - "id" : "5623928067739466034", - "kind" : "blogger#blog", - "locale" : dict, - "name" : "Julian Bunker Photography", - "pages" : int, - "posts" : int, - "published" : "2010-11-21T10:19:42-08:00", - "updated" : str, - "url" : "http://julianbphotography.blogspot.com/", - }, - "post": { - "author" : "Julian Bunker", - "content" : str, - "date" : "dt:2010-12-26 01:08:00", - "etag" : str, - "id" : "6955139236418998998", - "kind" : "blogger#post", - "published" : "2010-12-25T17:08:00-08:00", - "replies" : "0", - "title" : "Moon Rise", - "updated" : "2011-12-06T05:21:24-08:00", - "url" : "re:.+/2010/12/moon-rise.html$", - }, - "num": int, - "url": str, - }, - }), - ("blogger:http://www.julianbunker.com/2010/12/moon-rise.html"), - # video (#587) - (("http://cfnmscenesinmovies.blogspot.com/2011/11/" - "cfnm-scene-jenna-fischer-in-office.html"), { - "pattern": r"https://.+\.googlevideo\.com/videoplayback", - }), - # image URLs with width/height (#1061) - # ("https://aaaninja.blogspot.com/2020/08/altera-boob-press-2.html", { - # "pattern": r"https://1.bp.blogspot.com/.+/s0/altera_.+png", - # }), - # new image domain (#2204) - (("https://randomthingsthroughmyletterbox.blogspot.com/2022/01" - "/bitter-flowers-by-gunnar-staalesen-blog.html"), { - "pattern": r"https://blogger.googleusercontent.com/img/a/.+=s0$", - "count": 8, - }), - ) + pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)" + example = "https://BLOG.blogspot.com/1970/01/TITLE.html" def __init__(self, match): BloggerExtractor.__init__(self, match) @@ -161,17 +110,7 @@ class BloggerBlogExtractor(BloggerExtractor): """Extractor for an entire Blogger blog""" subcategory = "blog" pattern = BASE_PATTERN + r"/?$" - test = ( - ("https://julianbphotography.blogspot.com/", { - "range": "1-25", - "count": 25, - "pattern": r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg", - }), - ("blogger:https://www.kefblog.com.ng/", { - "range": "1-25", - "count": 25, - }), - ) + example = "https://BLOG.blogspot.com/" def posts(self, blog): return self.api.blog_posts(blog["id"]) @@ -181,12 +120,7 @@ class BloggerSearchExtractor(BloggerExtractor): """Extractor for Blogger search resuls""" subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)" - test = ( - ("https://julianbphotography.blogspot.com/search?q=400mm", { - "count": "< 10", - "keyword": {"query": "400mm"}, - }), - ) + example = "https://BLOG.blogspot.com/search?q=QUERY" def __init__(self, match): BloggerExtractor.__init__(self, match) @@ -203,13 +137,7 @@ class BloggerLabelExtractor(BloggerExtractor): """Extractor for Blogger posts by label""" subcategory = "label" pattern = BASE_PATTERN + r"/search/label/([^/?#]+)" - test = ( - ("https://dmmagazine.blogspot.com/search/label/D%26D", { - "range": "1-25", - "count": 25, - "keyword": {"label": "D&D"}, - }), - ) + example = "https://BLOG.blogspot.com/search/label/LABEL" def __init__(self, match): BloggerExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index d7b9b0c8..dc48090a 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -15,6 +15,7 @@ from urllib.parse import urlsplit, urlunsplit MEDIA_DOMAIN_OVERRIDES = { "cdn9.bunkr.ru" : "c9.bunkr.ru", "cdn12.bunkr.ru": "media-files12.bunkr.la", + "cdn-pizza.bunkr.ru": "pizza.bunkr.ru", } CDN_HOSTED_EXTENSIONS = ( @@ -28,53 +29,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): category = "bunkr" root = "https://bunkrr.su" pattern = r"(?:https?://)?(?:app\.)?bunkr+\.(?:la|[sr]u|is|to)/a/([^/?#]+)" - test = ( - ("https://bunkrr.su/a/Lktg9Keq", { - "pattern": r"https://cdn\.bunkr\.ru/test-テスト-\"&>-QjgneIQv\.png", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "album_id": "Lktg9Keq", - "album_name": 'test テスト "&>', - "count": 1, - "filename": 'test-テスト-"&>-QjgneIQv', - "id": "QjgneIQv", - "name": 'test-テスト-"&>', - "num": int, - }, - }), - # mp4 (#2239) - ("https://app.bunkr.ru/a/ptRHaCn2", { - "pattern": r"https://media-files\.bunkr\.ru/_-RnHoW69L\.mp4", - "content": "80e61d1dbc5896ae7ef9a28734c747b28b320471", - }), - # cdn4 - ("https://bunkr.is/a/iXTTc1o2", { - "pattern": r"https://(cdn|media-files)4\.bunkr\.ru/", - "content": "da29aae371b7adc8c5ef8e6991b66b69823791e8", - "keyword": { - "album_id": "iXTTc1o2", - "album_name": "test2", - "album_size": "691.1 KB", - "count": 2, - "description": "072022", - "filename": "re:video-wFO9FtxG|image-sZrQUeOx", - "id": "re:wFO9FtxG|sZrQUeOx", - "name": "re:video|image", - "num": int, - }, - }), - # cdn12 .ru TLD (#4147) - ("https://bunkrr.su/a/j1G29CnD", { - "pattern": r"https://(cdn12.bunkr.ru|media-files12.bunkr.la)/\w+", - "count": 8, - }), - ("https://bunkrr.su/a/Lktg9Keq"), - ("https://bunkr.la/a/Lktg9Keq"), - ("https://bunkr.su/a/Lktg9Keq"), - ("https://bunkr.ru/a/Lktg9Keq"), - ("https://bunkr.is/a/Lktg9Keq"), - ("https://bunkr.to/a/Lktg9Keq"), - ) + example = "https://bunkrr.su/a/ID" def fetch_album(self, album_id): # album metadata @@ -87,7 +42,6 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): cdn = None files = [] append = files.append - headers = {"Referer": self.root + "/"} pos = page.index('class="grid-images') for url in text.extract_iter(page, '", - }, - }), - ("https://www.catbox.moe/c/cd90s1"), - ("https://catbox.moe/c/w7tm47#"), - ) + example = "https://catbox.moe/c/ID" def metadata(self, page): extr = text.extract_from(page) @@ -62,15 +47,7 @@ class CatboxFileExtractor(Extractor): subcategory = "file" archive_fmt = "{filename}" pattern = r"(?:https?://)?(?:files|litter|de)\.catbox\.moe/([^/?#]+)" - test = ( - ("https://files.catbox.moe/8ih3y7.png", { - "pattern": r"^https://files\.catbox\.moe/8ih3y7\.png$", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "count": 1, - }), - ("https://litter.catbox.moe/t8v3n9.png"), - ("https://de.catbox.moe/bjdmz1.jpg"), - ) + example = "https://files.catbox.moe/NAME.EXT" def items(self): url = text.ensure_http_scheme(self.url) diff --git a/gallery_dl/extractor/comicvine.py b/gallery_dl/extractor/comicvine.py index 3a578864..d076795c 100644 --- a/gallery_dl/extractor/comicvine.py +++ b/gallery_dl/extractor/comicvine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -25,19 +25,7 @@ class ComicvineTagExtractor(BooruExtractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?comicvine\.gamespot\.com" r"(/([^/?#]+)/(\d+-\d+)/images/.*)") - test = ( - ("https://comicvine.gamespot.com/jock/4040-5653/images/", { - "pattern": r"https://comicvine\.gamespot\.com/a/uploads" - r"/original/\d+/\d+/\d+-.+\.(jpe?g|png)", - "count": ">= 140", - }), - (("https://comicvine.gamespot.com/batman/4005-1699" - "/images/?tag=Fan%20Art%20%26%20Cosplay"), { - "pattern": r"https://comicvine\.gamespot\.com/a/uploads" - r"/original/\d+/\d+/\d+-.+", - "count": ">= 450", - }), - ) + example = "https://comicvine.gamespot.com/TAG/123-45/images/" def __init__(self, match): BooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 660f9551..72c7b7c6 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -35,7 +35,6 @@ class Extractor(): cookies_domain = "" browser = None root = "" - test = None request_interval = 0.0 request_interval_min = 0.0 request_timestamp = 0.0 @@ -299,7 +298,7 @@ class Extractor(): useragent = self.config("user-agent") if useragent is None: useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " - "rv:115.0) Gecko/20100101 Firefox/115.0") + "rv:109.0) Gecko/20100101 Firefox/115.0") elif useragent == "browser": useragent = _browser_useragent() headers["User-Agent"] = useragent @@ -311,6 +310,13 @@ class Extractor(): else: headers["Accept-Encoding"] = "gzip, deflate" + custom_referer = self.config("referer", True) + if custom_referer: + if isinstance(custom_referer, str): + headers["Referer"] = custom_referer + elif self.root: + headers["Referer"] = self.root + "/" + custom_headers = self.config("headers") if custom_headers: headers.update(custom_headers) @@ -508,21 +514,6 @@ class Extractor(): result.append((Message.Queue, url, {"_extractor": extr})) return iter(result) - @classmethod - def _get_tests(cls): - """Yield an extractor's test cases as (URL, RESULTS) tuples""" - tests = cls.test - if not tests: - return - - if len(tests) == 2 and (not tests[1] or isinstance(tests[1], dict)): - tests = (tests,) - - for test in tests: - if isinstance(test, str): - test = (test, None) - yield test - @classmethod def _dump(cls, obj): util.dump_json(obj, ensure_ascii=False, indent=2) @@ -831,8 +822,8 @@ _browser_cookies = {} HTTP_HEADERS = { "firefox": ( - ("User-Agent", "Mozilla/5.0 ({}; rv:115.0) " - "Gecko/20100101 Firefox/115.0"), + ("User-Agent", "Mozilla/5.0 ({}; " + "rv:109.0) Gecko/20100101 Firefox/115.0"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," "image/avif,image/webp,*/*;q=0.8"), ("Accept-Language", "en-US,en;q=0.5"), diff --git a/gallery_dl/extractor/cyberdrop.py b/gallery_dl/extractor/cyberdrop.py index 7a79eca6..59fd1e58 100644 --- a/gallery_dl/extractor/cyberdrop.py +++ b/gallery_dl/extractor/cyberdrop.py @@ -14,32 +14,7 @@ class CyberdropAlbumExtractor(lolisafe.LolisafeAlbumExtractor): category = "cyberdrop" root = "https://cyberdrop.me" pattern = r"(?:https?://)?(?:www\.)?cyberdrop\.(?:me|to)/a/([^/?#]+)" - test = ( - # images - ("https://cyberdrop.me/a/keKRjm4t", { - "pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$", - "keyword": { - "album_id": "keKRjm4t", - "album_name": "Fate (SFW)", - "album_size": 150069254, - "count": 62, - "date": "dt:2020-06-18 13:14:20", - "description": "", - "id": r"re:\w{8}", - }, - }), - # videos - ("https://cyberdrop.to/a/l8gIAXVD", { - "pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$", - "count": 31, - "keyword": { - "album_id": "l8gIAXVD", - "album_name": "Achelois17 videos", - "album_size": 652037121, - "date": "dt:2020-06-16 15:40:44", - }, - }), - ) + example = "https://cyberdrop.me/a/ID" def fetch_album(self, album_id): url = self.root + "/a/" + self.album_id diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index b16d27a2..f69b0e25 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -173,38 +173,7 @@ class DanbooruTagExtractor(DanbooruExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=([^&#]*)" - test = ( - ("https://danbooru.donmai.us/posts?tags=bonocho", { - "content": "b196fb9f1668109d7774a0a82efea3ffdda07746", - }), - # test page transitions - ("https://danbooru.donmai.us/posts?tags=mushishi", { - "count": ">= 300", - }), - # 'external' option (#1747) - ("https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", { - "options": (("external", True),), - "pattern": r"https://i\.pximg\.net/img-original/img" - r"/2008/08/28/02/35/48/1476533_p0\.jpg", - }), - ("https://booru.allthefallen.moe/posts?tags=yume_shokunin", { - "count": 12, - }), - ("https://aibooru.online/posts?tags=center_frills&z=1", { - "pattern": r"https://cdn\.aibooru\.online/original" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "count": ">= 3", - }), - ("https://booru.borvar.art/posts?tags=chibi&z=1", { - "pattern": r"https://booru\.borvar\.art/data/original" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "count": ">= 3", - }), - ("https://hijiribe.donmai.us/posts?tags=bonocho"), - ("https://sonohara.donmai.us/posts?tags=bonocho"), - ("https://safebooru.donmai.us/posts?tags=bonocho"), - ("https://safe.aibooru.online/posts?tags=center_frills"), - ) + example = "https://danbooru.donmai.us/posts?tags=TAG" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -238,21 +207,7 @@ class DanbooruPoolExtractor(DanbooruExtractor): directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") archive_fmt = "p_{pool[id]}_{id}" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" - test = ( - ("https://danbooru.donmai.us/pools/7659", { - "content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", - }), - ("https://booru.allthefallen.moe/pools/9", { - "url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", - "count": 6, - }), - ("https://booru.borvar.art/pools/2", { - "url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", - "count": 4, - }), - ("https://aibooru.online/pools/1"), - ("https://danbooru.donmai.us/pool/show/7659"), - ) + example = "https://danbooru.donmai.us/pools/12345" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -275,26 +230,7 @@ class DanbooruPostExtractor(DanbooruExtractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" - test = ( - ("https://danbooru.donmai.us/posts/294929", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "keyword": {"date": "dt:2008-08-12 04:46:05"}, - }), - ("https://danbooru.donmai.us/posts/3613024", { - "pattern": r"https?://.+\.zip$", - "options": (("ugoira", True),) - }), - ("https://booru.allthefallen.moe/posts/22", { - "content": "21dda68e1d7e0a554078e62923f537d8e895cac8", - }), - ("https://aibooru.online/posts/1", { - "content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", - }), - ("https://booru.borvar.art/posts/1487", { - "content": "91273ac1ea413a12be468841e2b5804656a50bff", - }), - ("https://danbooru.donmai.us/post/show/294929"), - ) + example = "https://danbooru.donmai.us/posts/12345" def __init__(self, match): DanbooruExtractor.__init__(self, match) @@ -315,17 +251,7 @@ class DanbooruPopularExtractor(DanbooruExtractor): directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" pattern = BASE_PATTERN + r"/(?:explore/posts/)?popular(?:\?([^#]*))?" - test = ( - ("https://danbooru.donmai.us/explore/posts/popular"), - (("https://danbooru.donmai.us/explore/posts/popular" - "?date=2013-06-06&scale=week"), { - "range": "1-120", - "count": 120, - }), - ("https://booru.allthefallen.moe/explore/posts/popular"), - ("https://aibooru.online/explore/posts/popular"), - ("https://booru.borvar.art/explore/posts/popular"), - ) + example = "https://danbooru.donmai.us/explore/posts/popular" def __init__(self, match): DanbooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/desktopography.py b/gallery_dl/extractor/desktopography.py index 363341a5..35bb2992 100644 --- a/gallery_dl/extractor/desktopography.py +++ b/gallery_dl/extractor/desktopography.py @@ -23,7 +23,7 @@ class DesktopographySiteExtractor(DesktopographyExtractor): """Extractor for all desktopography exhibitions """ subcategory = "site" pattern = BASE_PATTERN + r"/$" - test = ("https://desktopography.net/",) + example = "https://desktopography.net/" def items(self): page = self.request(self.root).text @@ -42,7 +42,7 @@ class DesktopographyExhibitionExtractor(DesktopographyExtractor): """Extractor for a yearly desktopography exhibition""" subcategory = "exhibition" pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/" - test = ("https://desktopography.net/exhibition-2020/",) + example = "https://desktopography.net/exhibition-2020/" def __init__(self, match): DesktopographyExtractor.__init__(self, match) @@ -71,7 +71,7 @@ class DesktopographyEntryExtractor(DesktopographyExtractor): """Extractor for all resolutions of a desktopography wallpaper""" subcategory = "entry" pattern = BASE_PATTERN + r"/portfolios/([\w-]+)" - test = ("https://desktopography.net/portfolios/new-era/",) + example = "https://desktopography.net/portfolios/NAME/" def __init__(self, match): DesktopographyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 3302afbe..94c7f005 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -440,18 +440,7 @@ class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" subcategory = "user" pattern = BASE_PATTERN + r"/?$" - test = ( - ("https://www.deviantart.com/shimoda7", { - "pattern": r"/shimoda7/gallery$", - }), - ("https://www.deviantart.com/shimoda7", { - "options": (("include", "all"),), - "pattern": r"/shimoda7/" - r"(gallery(/scraps)?|posts(/statuses)?|favourites)$", - "count": 5, - }), - ("https://shimoda7.deviantart.com/"), - ) + example = "https://www.deviantart.com/USER" def initialize(self): pass @@ -475,84 +464,7 @@ class DeviantartGalleryExtractor(DeviantartExtractor): subcategory = "gallery" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/", { - "pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.(jpg|png)\?token=.+", - "count": ">= 30", - "keyword": { - "allows_comments": bool, - "author": { - "type": "regular", - "usericon": str, - "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", - "username": "shimoda7", - }, - "category_path": str, - "content": { - "filesize": int, - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "da_category": str, - "date": "type:datetime", - "deviationid": str, - "?download_filesize": int, - "extension": str, - "index": int, - "is_deleted": bool, - "is_downloadable": bool, - "is_favourited": bool, - "is_mature": bool, - "preview": { - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "published_time": int, - "stats": { - "comments": int, - "favourites": int, - }, - "target": dict, - "thumbs": list, - "title": str, - "url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", - "username": "shimoda7", - }, - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery", { - "pattern": r"https://www.deviantart.com/yakuzafc/gallery" - r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", - "count": ">= 15", - }), - # 'folders' option (#276) - ("https://www.deviantart.com/justatest235723/gallery", { - "count": 3, - "options": (("metadata", 1), ("folders", 1), ("original", 0)), - "keyword": { - "description": str, - "folders": list, - "is_watching": bool, - "license": str, - "tags": list, - }, - }), - ("https://www.deviantart.com/shimoda8/gallery/", { - "exception": exception.NotFoundError, - }), - - ("https://www.deviantart.com/shimoda7/gallery"), - ("https://www.deviantart.com/shimoda7/gallery/all"), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=/"), - ("https://shimoda7.deviantart.com/gallery/"), - ("https://shimoda7.deviantart.com/gallery/all/"), - ("https://shimoda7.deviantart.com/gallery/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/gallery/" def deviations(self): if self.flat and not self.group: @@ -567,32 +479,7 @@ class DeviantartFolderExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "{folder[title]}") archive_fmt = "F_{folder[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)/([^/?#]+)" - test = ( - # user - ("https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", { - "count": 5, - "options": (("original", False),), - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", { - "count": ">= 4", - "options": (("original", False),), - }), - # uuid - (("https://www.deviantart.com/shimoda7/gallery" - "/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc"), { - "count": 5, - "options": (("original", False),), - }), - # name starts with '_', special characters (#1451) - (("https://www.deviantart.com/justatest235723" - "/gallery/69302698/-test-b-c-d-e-f-"), { - "count": 1, - "options": (("original", False),), - }), - ("https://shimoda7.deviantart.com/gallery/722019/Miscellaneous"), - ("https://yakuzafc.deviantart.com/gallery/37412168/Crafts"), - ) + example = "https://www.deviantart.com/USER/gallery/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -621,33 +508,7 @@ class DeviantartStashExtractor(DeviantartExtractor): subcategory = "stash" archive_fmt = "{index}.{extension}" pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)" - test = ( - ("https://sta.sh/022c83odnaxc", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.png\?token=.+", - "content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", - "count": 1, - }), - # multiple stash items - ("https://sta.sh/21jf51j7pzl2", { - "options": (("original", False),), - "count": 4, - }), - # downloadable, but no "content" field (#307) - ("https://sta.sh/024t4coz16mi", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.rar\?token=.+", - "count": 1, - }), - # mixed folders and images (#659) - ("https://sta.sh/215twi387vfj", { - "options": (("original", False),), - "count": 4, - }), - ("https://sta.sh/abcdefghijkl", { - "count": 0, - }), - ) + example = "https://sta.sh/abcde" skip = Extractor.skip @@ -692,20 +553,7 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" - test = ( - ("https://www.deviantart.com/h3813067/favourites/", { - "options": (("metadata", True), ("flat", False)), # issue #271 - "count": 1, - }), - ("https://www.deviantart.com/h3813067/favourites/", { - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/h3813067/favourites/all"), - ("https://www.deviantart.com/h3813067/favourites/?catpath=/"), - ("https://h3813067.deviantart.com/favourites/"), - ("https://h3813067.deviantart.com/favourites/all"), - ("https://h3813067.deviantart.com/favourites/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/favourites/" def deviations(self): if self.flat: @@ -722,20 +570,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor): "{collection[title]}") archive_fmt = "C_{collection[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" - test = ( - (("https://www.deviantart.com/pencilshadings/favourites" - "/70595441/3D-Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - (("https://www.deviantart.com/pencilshadings/favourites" - "/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - ("https://pencilshadings.deviantart.com" - "/favourites/70595441/3D-Favorites"), - ) + example = "https://www.deviantart.com/USER/favourites/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -766,24 +601,7 @@ class DeviantartJournalExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Journal") archive_fmt = "j_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/(?:posts(?:/journals)?|journal)/?(?:\?.*)?$" - test = ( - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", - "options": (("journals", "text"),), - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "count": 0, - "options": (("journals", "none"),), - }), - ("https://www.deviantart.com/shimoda7/posts/"), - ("https://www.deviantart.com/shimoda7/journal/"), - ("https://www.deviantart.com/shimoda7/journal/?catpath=/"), - ("https://shimoda7.deviantart.com/journal/"), - ("https://shimoda7.deviantart.com/journal/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/posts/journals/" def deviations(self): return self.api.browse_user_journals(self.user, self.offset) @@ -796,45 +614,7 @@ class DeviantartStatusExtractor(DeviantartExtractor): filename_fmt = "{category}_{index}_{title}_{date}.{extension}" archive_fmt = "S_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/posts/statuses" - test = ( - ("https://www.deviantart.com/t1na/posts/statuses", { - "count": 0, - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "count": 4, - "url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074", - }), - # shared deviation - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "none"),), - "count": 1, - "pattern": r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token=", - }), - # shared sta.sh item - ("https://www.deviantart.com/vanillaghosties/posts/statuses", { - "options": (("journals", "none"), ("original", False)), - "range": "5-", - "count": 1, - "keyword": { - "index" : int, - "index_base36": "re:^[0-9a-z]+$", - "url" : "re:^https://sta.sh", - }, - }), - # "deleted" deviations in 'items' - ("https://www.deviantart.com/AndrejSKalin/posts/statuses", { - "options": (("journals", "none"), ("original", 0), - ("image-filter", "deviationid[:8] == '147C8B03'")), - "count": 2, - "archive": False, - "keyword": {"deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655"} - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "text"),), - "url": "c8744f7f733a3029116607b826321233c5ca452d", - }), - ) + example = "https://www.deviantart.com/USER/posts/statuses/" def deviations(self): for status in self.api.user_statuses(self.user, self.offset): @@ -898,19 +678,7 @@ class DeviantartPopularExtractor(DeviantartExtractor): r"(?:deviations/?)?\?order=(popular-[^/?#]+)" r"|((?:[\w-]+/)*)(popular-[^/?#]+)" r")/?(?:\?([^#]*))?") - test = ( - ("https://www.deviantart.com/?order=popular-all-time", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/popular-24-hours/?q=tree+house", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/artisan/popular-all-time/?q=tree"), - ) + example = "https://www.deviantart.com/popular-24-hours/" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -955,11 +723,7 @@ class DeviantartTagExtractor(DeviantartExtractor): directory_fmt = ("{category}", "Tags", "{search_tags}") archive_fmt = "T_{search_tags}_{index}.{extension}" pattern = r"(?:https?://)?www\.deviantart\.com/tag/([^/?#]+)" - test = ("https://www.deviantart.com/tag/nature", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }) + example = "https://www.deviantart.com/tag/TAG" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -978,10 +742,7 @@ class DeviantartWatchExtractor(DeviantartExtractor): subcategory = "watch" pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" r"/(?:watch/deviations|notifications/watch)()()") - test = ( - ("https://www.deviantart.com/watch/deviations"), - ("https://www.deviantart.com/notifications/watch"), - ) + example = "https://www.deviantart.com/watch/deviations" def deviations(self): return self.api.browse_deviantsyouwatch() @@ -991,7 +752,7 @@ class DeviantartWatchPostsExtractor(DeviantartExtractor): """Extractor for Posts from watched users""" subcategory = "watch-posts" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" - test = ("https://www.deviantart.com/watch/posts",) + example = "https://www.deviantart.com/watch/posts" def deviations(self): return self.api.browse_posts_deviantsyouwatch() @@ -1009,100 +770,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor): r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" r"(\d+)" # bare deviation ID without slug r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 - test = ( - (("https://www.deviantart.com/shimoda7/art/For-the-sake-10073852"), { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/zzz/art/zzz-1234567890", { - "exception": exception.NotFoundError, - }), - (("https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576"), { - "options": (("comments", True),), - "keyword": {"comments": list}, - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.jpg\?token=.+", - }), - # wixmp URL rewrite - (("https://www.deviantart.com/citizenfresh/art/Hverarond-789295466"), { - "pattern": (r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token="), - }), - # GIF (#242) - (("https://www.deviantart.com/skatergators/art/COM-Moni-781571783"), { - "pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-" - r"8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-" - r"f30961955bec\.gif\?token=ey...", - }), - # Flash animation with GIF preview (#1731) - ("https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.swf\?token=.+", - "keyword": { - "filename": "flash_comic_tutorial_by_yuumei-d3juatd", - "extension": "swf", - }, - }), - # sta.sh URLs from description (#302) - (("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), { - "options": (("extra", 1), ("original", 0)), - "pattern": DeviantartStashExtractor.pattern, - "range": "2-", - "count": 4, - }), - # sta.sh URL from deviation["text_content"]["body"]["features"] - (("https://www.deviantart.com" - "/cimar-wildehopps/art/Honorary-Vixen-859809305"), { - "options": (("extra", 1),), - "pattern": ("text:\n|" + - DeviantartStashExtractor.pattern), - "count": 2, - }), - # journal - ("https://www.deviantart.com/shimoda7/journal/ARTility-583755752", { - "url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", - "pattern": "text:\n", - }), - # journal-like post with isJournal == False (#419) - ("https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", { - "url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", - "pattern": "text:\n", - }), - # /view/ URLs - ("https://deviantart.com/view/904858796/", { - "content": "8770ec40ad1c1d60f6b602b16301d124f612948f", - }), - ("http://www.deviantart.com/view/890672057", { - "content": "1497e13d925caeb13a250cd666b779a640209236", - }), - ("https://www.deviantart.com/view/706871727", { - "content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65", - }), - ("https://www.deviantart.com/view/1", { - "exception": exception.NotFoundError, - }), - # /deviation/ (#3558) - ("https://www.deviantart.com/deviation/817215762"), - # fav.me (#3558) - ("https://fav.me/ddijrpu", { - "count": 1, - }), - ("https://fav.me/dddd", { - "exception": exception.NotFoundError, - }), - # old-style URLs - ("https://shimoda7.deviantart.com" - "/art/For-the-sake-of-a-memory-10073852"), - ("https://myria-moon.deviantart.com" - "/art/Aime-Moi-part-en-vadrouille-261986576"), - ("https://zzz.deviantart.com/art/zzz-1234567890"), - # old /view/ URLs from the Wayback Machine - ("https://www.deviantart.com/view.php?id=14864502"), - ("http://www.deviantart.com/view-full.php?id=100842"), - - ("https://www.fxdeviantart.com/zzz/art/zzz-1234567890"), - ("https://www.fxdeviantart.com/view/1234567890"), - ) + example = "https://www.deviantart.com/UsER/art/TITLE-12345" skip = Extractor.skip @@ -1133,13 +801,7 @@ class DeviantartScrapsExtractor(DeviantartExtractor): archive_fmt = "s_{_username}_{index}.{extension}" cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/(?:\?catpath=)?scraps\b" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/scraps", { - "count": 12, - }), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=scraps"), - ("https://shimoda7.deviantart.com/gallery/?catpath=scraps"), - ) + example = "https://www.deviantart.com/USER/gallery/scraps" def deviations(self): self.login() @@ -1157,11 +819,7 @@ class DeviantartSearchExtractor(DeviantartExtractor): cookies_domain = ".deviantart.com" pattern = (r"(?:https?://)?www\.deviantart\.com" r"/search(?:/deviations)?/?\?([^#]+)") - test = ( - ("https://www.deviantart.com/search?q=tree"), - ("https://www.deviantart.com/search/deviations?order=popular-1-week"), - ) - + example = "https://www.deviantart.com/search?q=QUERY" skip = Extractor.skip def __init__(self, match): @@ -1212,13 +870,7 @@ class DeviantartGallerySearchExtractor(DeviantartExtractor): archive_fmt = "g_{_username}_{index}.{extension}" cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" - test = ( - ("https://www.deviantart.com/shimoda7/gallery?q=memory", { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular"), - ) + example = "https://www.deviantart.com/USER/gallery?q=QUERY" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -1250,11 +902,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor): """Extractor for user's watched users""" subcategory = "following" pattern = BASE_PATTERN + "/about#watching$" - test = ("https://www.deviantart.com/shimoda7/about#watching", { - "pattern": DeviantartUserExtractor.pattern, - "range": "1-50", - "count": 50, - }) + example = "https://www.deviantart.com/USER/about#watching" def items(self): eclipse_api = DeviantartEclipseAPI(self) @@ -1774,11 +1422,9 @@ class DeviantartEclipseAPI(): def _call(self, endpoint, params): url = "https://www.deviantart.com/_napi" + endpoint - headers = {"Referer": "https://www.deviantart.com/"} params["csrf_token"] = self.csrf_token or self._fetch_csrf_token() - response = self.request( - url, params=params, headers=headers, fatal=None) + response = self.request(url, params=params, fatal=None) if response.status_code == 404: raise exception.StopExtraction( diff --git a/gallery_dl/extractor/directlink.py b/gallery_dl/extractor/directlink.py index e85eb8db..26f21849 100644 --- a/gallery_dl/extractor/directlink.py +++ b/gallery_dl/extractor/directlink.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,36 +20,7 @@ class DirectlinkExtractor(Extractor): pattern = (r"(?i)https?://(?P[^/?#]+)/(?P[^?#]+\." r"(?:jpe?g|jpe|png|gif|web[mp]|mp4|mkv|og[gmv]|opus))" r"(?:\?(?P[^#]*))?(?:#(?P.*))?$") - test = ( - (("https://en.wikipedia.org/static/images/project-logos/enwiki.png"), { - "url": "18c5d00077332e98e53be9fed2ee4be66154b88d", - "keyword": "105770a3f4393618ab7b811b731b22663b5d3794", - }), - # empty path - (("https://example.org/file.webm"), { - "url": "2d807ed7059d1b532f1bb71dc24b510b80ff943f", - "keyword": "29dad729c40fb09349f83edafa498dba1297464a", - }), - # more complex example - ("https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", { - "url": "6fb1061390f8aada3db01cb24b51797c7ee42b31", - "keyword": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", - }), - # percent-encoded characters - ("https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", { - "url": "2627e8140727fdf743f86fe18f69f99a052c9718", - "keyword": "831790fddda081bdddd14f96985ab02dc5b5341f", - }), - # upper case file extension (#296) - ("https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw" - ".JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpP" - "mZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG"), - # internationalized domain name - ("https://räksmörgås.josefsson.org/raksmorgas.jpg", { - "url": "a65667f670b194afbd1e3ea5e7a78938d36747da", - "keyword": "fd5037fe86eebd4764e176cbaf318caec0f700be", - }), - ) + example = "https://en.wikipedia.org/static/images/project-logos/enwiki.png" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index 59e8c90d..733d0d8f 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -43,18 +43,7 @@ class DynastyscansBase(): class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): """Extractor for manga-chapters from dynasty-scans.com""" pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)" - test = ( - (("http://dynasty-scans.com/chapters/" - "hitoribocchi_no_oo_seikatsu_ch33"), { - "url": "dce64e8c504118f1ab4135c00245ea12413896cb", - "keyword": "b67599703c27316a2fe4f11c3232130a1904e032", - }), - (("http://dynasty-scans.com/chapters/" - "new_game_the_spinoff_special_13"), { - "url": "dbe5bbb74da2edcfb1832895a484e2a40bc8b538", - "keyword": "6b674eb3a274999153f6be044973b195008ced2f", - }), - ) + example = "https://dynasty-scans.com/chapters/NAME" def metadata(self, page): extr = text.extract_from(page) @@ -93,10 +82,7 @@ class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): chapterclass = DynastyscansChapterExtractor reverse = False pattern = BASE_PATTERN + r"(/series/[^/?#]+)" - test = ("https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", { - "pattern": DynastyscansChapterExtractor.pattern, - "count": ">= 100", - }) + example = "https://dynasty-scans.com/series/NAME" def chapters(self, page): return [ @@ -112,16 +98,7 @@ class DynastyscansSearchExtractor(DynastyscansBase, Extractor): filename_fmt = "{image_id}.{extension}" archive_fmt = "i_{image_id}" pattern = BASE_PATTERN + r"/images/?(?:\?([^#]+))?$" - test = ( - ("https://dynasty-scans.com/images?with[]=4930&with[]=5211", { - "url": "22cf0fb64e12b29e79b0a3d26666086a48f9916a", - "keyword": "11cbc555a15528d25567977b8808e10369c4c3ee", - }), - ("https://dynasty-scans.com/images", { - "range": "1", - "count": 1, - }), - ) + example = "https://dynasty-scans.com/images?QUERY" def __init__(self, match): Extractor.__init__(self, match) @@ -150,10 +127,7 @@ class DynastyscansImageExtractor(DynastyscansSearchExtractor): """Extractor for individual images on dynasty-scans.com""" subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" - test = ("https://dynasty-scans.com/images/1245", { - "url": "15e54bd94148a07ed037f387d046c27befa043b2", - "keyword": "0d8976c2d6fbc9ed6aa712642631b96e456dc37f", - }) + example = "https://dynasty-scans.com/images/12345" def images(self): return (self.query,) diff --git a/gallery_dl/extractor/e621.py b/gallery_dl/extractor/e621.py index cb1aea40..af963bca 100644 --- a/gallery_dl/extractor/e621.py +++ b/gallery_dl/extractor/e621.py @@ -84,48 +84,13 @@ BASE_PATTERN = E621Extractor.update({ class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor): """Extractor for e621 posts from tag searches""" pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)" - test = ( - ("https://e621.net/posts?tags=anry", { - "url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", - "content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", - }), - ("https://e621.net/post/index/1/anry"), - ("https://e621.net/post?tags=anry"), - - ("https://e926.net/posts?tags=anry", { - "url": "12198b275c62ffe2de67cca676c8e64de80c425d", - "content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", - }), - ("https://e926.net/post/index/1/anry"), - ("https://e926.net/post?tags=anry"), - - ("https://e6ai.net/posts?tags=anry"), - ("https://e6ai.net/post/index/1/anry"), - ("https://e6ai.net/post?tags=anry"), - ) + example = "https://e621.net/posts?tags=TAG" class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): """Extractor for e621 pools""" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" - test = ( - ("https://e621.net/pools/73", { - "url": "1bd09a72715286a79eea3b7f09f51b3493eb579a", - "content": "91abe5d5334425d9787811d7f06d34c77974cd22", - }), - ("https://e621.net/pool/show/73"), - - ("https://e926.net/pools/73", { - "url": "6936f1b6a18c5c25bee7cad700088dbc2503481b", - "content": "91abe5d5334425d9787811d7f06d34c77974cd22", - }), - ("https://e926.net/pool/show/73"), - - ("https://e6ai.net/pools/3", { - "url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", - }), - ("https://e6ai.net/pool/show/3"), - ) + example = "https://e621.net/pools/12345" def posts(self): self.log.info("Fetching posts of pool %s", self.pool_id) @@ -151,67 +116,7 @@ class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): """Extractor for single e621 posts""" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" - test = ( - ("https://e621.net/posts/535", { - "url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", - "content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - "keyword": {"date": "dt:2007-02-17 19:02:32"}, - }), - ("https://e621.net/posts/3181052", { - "options": (("metadata", "notes,pools"),), - "pattern": r"https://static\d\.e621\.net/data/c6/8c" - r"/c68cca0643890b615f75fb2719589bff\.png", - "keyword": { - "notes": [ - { - "body": "Little Legends 2", - "created_at": "2022-05-16T13:58:38.877-04:00", - "creator_id": 517450, - "creator_name": "EeveeCuddler69", - "height": 475, - "id": 321296, - "is_active": True, - "post_id": 3181052, - "updated_at": "2022-05-16T13:59:02.050-04:00", - "version": 3, - "width": 809, - "x": 83, - "y": 117, - }, - ], - "pools": [ - { - "category": "series", - "created_at": "2022-02-17T00:29:22.669-05:00", - "creator_id": 1077440, - "creator_name": "Yeetus90", - "description": "* \"Little Legends\":/pools/27971\r\n" - "* Little Legends 2\r\n" - "* \"Little Legends 3\":/pools/27481", - "id": 27492, - "is_active": False, - "name": "Little Legends 2", - "post_count": 39, - "post_ids": list, - "updated_at": "2022-03-27T06:30:03.382-04:00" - }, - ], - }, - }), - ("https://e621.net/post/show/535"), - - ("https://e926.net/posts/535", { - "url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", - "content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - }), - ("https://e926.net/post/show/535"), - - ("https://e6ai.net/posts/23", { - "url": "3c85a806b3d9eec861948af421fe0e8ad6b8f881", - "content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", - }), - ("https://e6ai.net/post/show/23"), - ) + example = "https://e621.net/posts/12345" def posts(self): url = "{}/posts/{}.json".format(self.root, self.post_id) @@ -221,23 +126,7 @@ class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor): """Extractor for popular images from e621""" pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" - test = ( - ("https://e621.net/explore/posts/popular"), - (("https://e621.net/explore/posts/popular" - "?date=2019-06-01&scale=month"), { - "pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", - "count": ">= 70", - }), - - ("https://e926.net/explore/posts/popular"), - (("https://e926.net/explore/posts/popular" - "?date=2019-06-01&scale=month"), { - "pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", - "count": ">= 70", - }), - - ("https://e6ai.net/explore/posts/popular"), - ) + example = "https://e621.net/explore/posts/popular" def posts(self): return self._pagination("/popular.json", self.params) @@ -249,21 +138,7 @@ class E621FavoriteExtractor(E621Extractor): directory_fmt = ("{category}", "Favorites", "{user_id}") archive_fmt = "f_{user_id}_{id}" pattern = BASE_PATTERN + r"/favorites(?:\?([^#]*))?" - test = ( - ("https://e621.net/favorites"), - ("https://e621.net/favorites?page=2&user_id=53275", { - "pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", - "count": "> 260", - }), - - ("https://e926.net/favorites"), - ("https://e926.net/favorites?page=2&user_id=53275", { - "pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", - "count": "> 260", - }), - - ("https://e6ai.net/favorites"), - ) + example = "https://e621.net/favorites" def __init__(self, match): E621Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/erome.py b/gallery_dl/extractor/erome.py index cb527410..2aed678f 100644 --- a/gallery_dl/extractor/erome.py +++ b/gallery_dl/extractor/erome.py @@ -91,29 +91,7 @@ class EromeAlbumExtractor(EromeExtractor): """Extractor for albums on erome.com""" subcategory = "album" pattern = BASE_PATTERN + r"/a/(\w+)" - test = ( - ("https://www.erome.com/a/NQgdlWvk", { - "pattern": r"https://v\d+\.erome\.com/\d+" - r"/NQgdlWvk/j7jlzmYB_480p\.mp4", - "count": 1, - "keyword": { - "album_id": "NQgdlWvk", - "num": 1, - "title": "porn", - "user": "yYgWBZw8o8qsMzM", - }, - }), - ("https://www.erome.com/a/TdbZ4ogi", { - "pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", - "count": 6, - "keyword": { - "album_id": "TdbZ4ogi", - "num": int, - "title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", - "user": "yYgWBZw8o8qsMzM", - }, - }), - ) + example = "https://www.erome.com/a/ID" def albums(self): return (self.item,) @@ -122,10 +100,7 @@ class EromeAlbumExtractor(EromeExtractor): class EromeUserExtractor(EromeExtractor): subcategory = "user" pattern = BASE_PATTERN + r"/(?!a/|search\?)([^/?#]+)" - test = ("https://www.erome.com/yYgWBZw8o8qsMzM", { - "range": "1-25", - "count": 25, - }) + example = "https://www.erome.com/USER" def albums(self): url = "{}/{}".format(self.root, self.item) @@ -135,10 +110,7 @@ class EromeUserExtractor(EromeExtractor): class EromeSearchExtractor(EromeExtractor): subcategory = "search" pattern = BASE_PATTERN + r"/search\?q=([^&#]+)" - test = ("https://www.erome.com/search?q=cute", { - "range": "1-25", - "count": 25, - }) + example = "https://www.erome.com/search?q=QUERY" def albums(self): url = self.root + "/search" diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index d5f1d02b..44bfe7d8 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -45,7 +45,6 @@ class ExhentaiExtractor(Extractor): if self.version != "ex": self.cookies.set("nw", "1", domain=self.cookies_domain) - self.session.headers["Referer"] = self.root + "/" self.original = self.config("original", True) limits = self.config("limits", False) @@ -109,61 +108,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): pattern = (BASE_PATTERN + r"(?:/g/(\d+)/([\da-f]{10})" r"|/s/([\da-f]{10})/(\d+)-(\d+))") - test = ( - ("https://exhentai.org/g/1200119/d55c44d3d0/", { - "options": (("original", False),), - "keyword": { - "cost": int, - "date": "dt:2018-03-18 20:14:00", - "eh_category": "Non-H", - "expunged": False, - "favorites": r"re:^[12]\d$", - "filecount": "4", - "filesize": 1488978, - "gid": 1200119, - "height": int, - "image_token": "re:[0-9a-f]{10}", - "lang": "ja", - "language": "Japanese", - "parent": "", - "rating": r"re:\d\.\d+", - "size": int, - "tags": [ - "parody:komi-san wa komyushou desu.", - "character:shouko komi", - "group:seventh lowlife", - "other:sample", - ], - "thumb": "https://exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f8" - "3bcb1630ab1350640-624622-736-1036-jpg_250.jpg", - "title": "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan de" - "su (Komi-san wa Komyushou desu) [Sample]", - "title_jpn": "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆" - "です。 (古見さんは、コミュ症です。) [見本]", - "token": "d55c44d3d0", - "torrentcount": "0", - "uploader": "klorpa", - "width": int, - }, - "content": ("2c68cff8a7ca540a78c36fdbf5fbae0260484f87", - "e9891a4c017ed0bb734cd1efba5cd03f594d31ff"), - }), - ("https://exhentai.org/g/960461/4f0e369d82/", { - "exception": exception.NotFoundError, - }), - ("http://exhentai.org/g/962698/7f02358e00/", { - "exception": exception.AuthorizationError, - }), - ("https://exhentai.org/s/f68367b4c8/1200119-3", { - "options": (("original", False),), - "count": 2, - }), - ("https://e-hentai.org/s/f68367b4c8/1200119-3", { - "options": (("original", False),), - "count": 2, - }), - ("https://g.e-hentai.org/g/1200119/d55c44d3d0/"), - ) + example = "https://e-hentai.org/g/12345/67890abcde/" def __init__(self, match): ExhentaiExtractor.__init__(self, match) @@ -179,6 +124,20 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): if source == "hitomi": self.items = self._items_hitomi + def favorite(self, slot="0"): + url = self.root + "/gallerypopups.php" + params = { + "gid": self.gallery_id, + "t" : self.gallery_token, + "act": "addfav", + } + data = { + "favcat" : slot, + "apply" : "Apply Changes", + "update" : "1", + } + self.request(url, method="POST", params=params, data=data) + def items(self): self.login() @@ -223,6 +182,10 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): data["_http_validate"] = None yield Message.Url, url, data + fav = self.config("fav") + if fav is not None: + self.favorite(fav) + def _items_hitomi(self): if self.config("metadata", False): data = self.metadata_from_api() @@ -463,26 +426,10 @@ class ExhentaiSearchExtractor(ExhentaiExtractor): """Extractor for exhentai search results""" subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))" - test = ( - ("https://e-hentai.org/?f_search=touhou"), - ("https://exhentai.org/?f_cats=767&f_search=touhou"), - ("https://exhentai.org/tag/parody:touhou+project"), - (("https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0" - "&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0" - "&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter"), { - "pattern": ExhentaiGalleryExtractor.pattern, - "range": "1-30", - "count": 30, - "keyword": { - "gallery_id": int, - "gallery_token": r"re:^[0-9a-f]{10}$" - }, - }), - ) + example = "https://e-hentai.org/?f_search=QUERY" def __init__(self, match): ExhentaiExtractor.__init__(self, match) - self.search_url = self.root _, query, tag = match.groups() if tag: @@ -497,6 +444,9 @@ class ExhentaiSearchExtractor(ExhentaiExtractor): if "next" not in self.params: self.params["page"] = text.parse_int(self.params.get("page")) + def _init(self): + self.search_url = self.root + def items(self): self.login() data = {"_extractor": ExhentaiGalleryExtractor} @@ -533,15 +483,7 @@ class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor): """Extractor for favorited exhentai galleries""" subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?" - test = ( - ("https://e-hentai.org/favorites.php", { - "count": 1, - "pattern": r"https?://e-hentai\.org/g/1200119/d55c44d3d0" - }), - ("https://exhentai.org/favorites.php?favcat=1&f_search=touhou" - "&f_apply=Search+Favorites"), - ) + example = "https://e-hentai.org/favorites.php" - def __init__(self, match): - ExhentaiSearchExtractor.__init__(self, match) + def _init(self): self.search_url = self.root + "/favorites.php" diff --git a/gallery_dl/extractor/fallenangels.py b/gallery_dl/extractor/fallenangels.py index 0503dcfa..650a7075 100644 --- a/gallery_dl/extractor/fallenangels.py +++ b/gallery_dl/extractor/fallenangels.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -13,24 +13,11 @@ from .. import text, util class FallenangelsChapterExtractor(ChapterExtractor): - """Extractor for manga-chapters from fascans.com""" + """Extractor for manga chapters from fascans.com""" category = "fallenangels" pattern = (r"(?:https?://)?(manga|truyen)\.fascans\.com" r"/manga/([^/?#]+)/([^/?#]+)") - test = ( - ("https://manga.fascans.com/manga/chronos-ruler/20/1", { - "url": "4604a7914566cc2da0ff789aa178e2d1c8c241e3", - "keyword": "2dfcc50020e32cd207be88e2a8fac0933e36bdfb", - }), - ("http://truyen.fascans.com/manga/hungry-marie/8", { - "url": "1f923d9cb337d5e7bbf4323719881794a951c6ae", - "keyword": "2bdb7334c0e3eceb9946ffd3132df679b4a94f6a", - }), - ("http://manga.fascans.com/manga/rakudai-kishi-no-eiyuutan/19.5", { - "url": "273f6863966c83ea79ad5846a2866e08067d3f0e", - "keyword": "d1065685bfe0054c4ff2a0f20acb089de4cec253", - }), - ) + example = "https://manga.fascans.com/manga/NAME/CHAPTER/" def __init__(self, match): self.version, self.manga, self.chapter = match.groups() @@ -66,16 +53,7 @@ class FallenangelsMangaExtractor(MangaExtractor): chapterclass = FallenangelsChapterExtractor category = "fallenangels" pattern = r"(?:https?://)?((manga|truyen)\.fascans\.com/manga/[^/]+)/?$" - test = ( - ("https://manga.fascans.com/manga/chronos-ruler", { - "url": "eea07dd50f5bc4903aa09e2cc3e45c7241c9a9c2", - "keyword": "c414249525d4c74ad83498b3c59a813557e59d7e", - }), - ("https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", { - "url": "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b", - "keyword": "2d2a2a5d9ea5925eb9a47bb13d848967f3af086c", - }), - ) + example = "https://manga.fascans.com/manga/NAME" def __init__(self, match): url = "https://" + match.group(1) diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py index 921ddb62..4572bea6 100644 --- a/gallery_dl/extractor/fanbox.py +++ b/gallery_dl/extractor/fanbox.py @@ -10,7 +10,6 @@ from .common import Extractor, Message from .. import text import re - BASE_PATTERN = ( r"(?:https?://)?(?:" r"(?!www\.)([\w-]+)\.fanbox\.cc|" @@ -30,12 +29,12 @@ class FanboxExtractor(Extractor): def _init(self): self.embeds = self.config("embeds", True) - def items(self): if self._warning: if not self.cookies_check(("FANBOXSESSID",)): self.log.warning("no 'FANBOXSESSID' cookie set") FanboxExtractor._warning = False + def items(self): for content_body, post in self.posts(): yield Message.Directory, post yield from self._get_urls_from_post(content_body, post) @@ -243,20 +242,7 @@ class FanboxCreatorExtractor(FanboxExtractor): """Extractor for a Fanbox creator's works""" subcategory = "creator" pattern = BASE_PATTERN + r"(?:/posts)?/?$" - test = ( - ("https://xub.fanbox.cc", { - "range": "1-15", - "count": ">= 15", - "keyword": { - "creatorId" : "xub", - "tags" : list, - "title" : str, - }, - }), - ("https://xub.fanbox.cc/posts"), - ("https://www.fanbox.cc/@xub/"), - ("https://www.fanbox.cc/@xub/posts"), - ) + example = "https://USER.fanbox.cc/" def __init__(self, match): FanboxExtractor.__init__(self, match) @@ -271,55 +257,7 @@ class FanboxPostExtractor(FanboxExtractor): """Extractor for media from a single Fanbox post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.fanbox.cc/@xub/posts/1910054", { - "count": 3, - "keyword": { - "title": "えま★おうがすと", - "tags": list, - "hasAdultContent": True, - "isCoverImage": False - }, - }), - # entry post type, image embedded in html of the post - ("https://nekoworks.fanbox.cc/posts/915", { - "count": 2, - "keyword": { - "title": "【SAYORI FAN CLUB】お届け内容", - "tags": list, - "html": str, - "hasAdultContent": True - }, - }), - # article post type, imageMap, 2 twitter embeds, fanbox embed - ("https://steelwire.fanbox.cc/posts/285502", { - "options": (("embeds", True),), - "count": 10, - "keyword": { - "title": "イラスト+SS｜義足の炭鉱少年が義足を見せてくれるだけ 【全体公開版】", - "tags": list, - "articleBody": dict, - "hasAdultContent": True - }, - }), - # 'content' metadata (#3020) - ("https://www.fanbox.cc/@official-en/posts/4326303", { - "keyword": { - "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, " - r"September 5th, 2022, we are happy to announce " - r"the start of the FANBOX hashtag event " - r"#MySetupTour ! \nAbout the event\nTo join this " - r"event .+ \nPlease check this page for further " - r"details regarding the Privacy & Terms.\n" - r"https://fanbox.pixiv.help/.+/10184952456601\n\n\n" - r"Thank you for your continued support of FANBOX.$", - }, - }), - # imageMap file order (#2718) - ("https://mochirong.fanbox.cc/posts/3746116", { - "url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", - }), - ) + example = "https://USER.fanbox.cc/posts/12345" def __init__(self, match): FanboxExtractor.__init__(self, match) @@ -334,9 +272,7 @@ class FanboxRedirectExtractor(Extractor): category = "fanbox" subcategory = "redirect" pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)" - test = ("https://www.pixiv.net/fanbox/creator/52336352", { - "pattern": FanboxCreatorExtractor.pattern, - }) + example = "https://www.pixiv.net/fanbox/creator/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/fanleaks.py b/gallery_dl/extractor/fanleaks.py index 466bb8c9..886e8934 100644 --- a/gallery_dl/extractor/fanleaks.py +++ b/gallery_dl/extractor/fanleaks.py @@ -7,7 +7,7 @@ """Extractors for https://fanleaks.club/""" from .common import Extractor, Message -from .. import text, exception +from .. import text class FanleaksExtractor(Extractor): @@ -36,34 +36,10 @@ class FanleaksExtractor(Extractor): class FanleaksPostExtractor(FanleaksExtractor): - """Extractor for individual posts on fanleak.club""" + """Extractor for individual posts on fanleaks.club""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)" - test = ( - ("https://fanleaks.club/selti/880", { - "pattern": (r"https://fanleaks\.club//models" - r"/selti/images/selti_0880\.jpg"), - "keyword": { - "model_id": "selti", - "model" : "Selti", - "id" : 880, - "type" : "photo", - }, - }), - ("https://fanleaks.club/daisy-keech/1038", { - "pattern": (r"https://fanleaks\.club//models" - r"/daisy-keech/videos/daisy-keech_1038\.mp4"), - "keyword": { - "model_id": "daisy-keech", - "model" : "Daisy Keech", - "id" : 1038, - "type" : "video", - }, - }), - ("https://fanleaks.club/hannahowo/000", { - "exception": exception.NotFoundError, - }), - ) + example = "https://fanleaks.club/MODEL/12345" def __init__(self, match): FanleaksExtractor.__init__(self, match) @@ -79,22 +55,7 @@ class FanleaksModelExtractor(FanleaksExtractor): subcategory = "model" pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club" r"/(?!latest/?$)([^/?#]+)/?$") - test = ( - ("https://fanleaks.club/hannahowo", { - "pattern": (r"https://fanleaks\.club//models" - r"/hannahowo/(images|videos)/hannahowo_\d+\.\w+"), - "range" : "1-100", - "count" : 100, - }), - ("https://fanleaks.club/belle-delphine", { - "pattern": (r"https://fanleaks\.club//models" - r"/belle-delphine/(images|videos)" - r"/belle-delphine_\d+\.\w+"), - "range" : "1-100", - "count" : 100, - }), - ("https://fanleaks.club/daisy-keech"), - ) + example = "https://fanleaks.club/MODEL" def items(self): page_num = 1 @@ -102,8 +63,7 @@ class FanleaksModelExtractor(FanleaksExtractor): self.root + "/" + self.model_id, notfound="model").text data = { "model_id": self.model_id, - "model" : text.unescape( - text.extr(page, 'mt-4">', "

")), + "model" : text.unescape(text.extr(page, 'mt-4">', "")), "type" : "photo", } page_url = text.extr(page, "url: '", "'") diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index ca1eeef1..f1d51e21 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -22,7 +22,6 @@ class FantiaExtractor(Extractor): def _init(self): self.headers = { "Accept" : "application/json, text/plain, */*", - "Referer": self.root, "X-Requested-With": "XMLHttpRequest", } self._empty_plan = { @@ -65,11 +64,9 @@ class FantiaExtractor(Extractor): def _pagination(self, url): params = {"page": 1} - headers = self.headers.copy() - del headers["X-Requested-With"] while True: - page = self.request(url, params=params, headers=headers).text + page = self.request(url, params=params).text self._csrf_token(page) post_id = None @@ -173,17 +170,7 @@ class FantiaCreatorExtractor(FantiaExtractor): """Extractor for a Fantia creator's works""" subcategory = "creator" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)" - test = ( - ("https://fantia.jp/fanclubs/6939", { - "range": "1-25", - "count": ">= 25", - "keyword": { - "fanclub_user_id" : 52152, - "tags" : list, - "title" : str, - }, - }), - ) + example = "https://fantia.jp/fanclubs/12345" def __init__(self, match): FantiaExtractor.__init__(self, match) @@ -198,53 +185,7 @@ class FantiaPostExtractor(FantiaExtractor): """Extractor for media from a single Fantia post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)" - test = ( - ("https://fantia.jp/posts/1166373", { - "pattern": r"https://(" - r"c\.fantia\.jp/uploads/post/file/1166373/|" - r"cc\.fantia\.jp/uploads/post_content_photo" - r"/file/732549[01]|" - r"fantia\.jp/posts/1166373/album_image\?)", - "keyword": { - "blogpost_text": r"re:^$|" - r"This is a test.\n\nThis is a test.\n\n|" - r"Link to video:\nhttps://www.youtube.com" - r"/watch\?v=5SSdvNcAagI\n\nhtml img from " - r"another site:\n\n\n\n\n\n", - "comment": "\n\n", - "content_category": "re:thumb|blog|photo_gallery", - "content_comment": str, - "content_filename": "re:|", - "content_title": r"re:Test (Blog Content \d+|Image Gallery)" - r"|thumb", - "date": "dt:2022-03-09 16:46:12", - "fanclub_id": 356320, - "fanclub_name": "Test Fantia", - "fanclub_url": "https://fantia.jp/fanclubs/356320", - "fanclub_user_id": 7487131, - "fanclub_user_name": "2022/03/08 15:13:52の名無し", - "file_url": str, - "filename": str, - "num": int, - "plan": dict, - "post_id": 1166373, - "post_title": "Test Fantia Post", - "post_url": "https://fantia.jp/posts/1166373", - "posted_at": "Thu, 10 Mar 2022 01:46:12 +0900", - "rating": "general", - "tags": [], - }, - }), - ("https://fantia.jp/posts/508363", { - "count": 6, - "keyword": { - "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", - "tags": list, - "rating": "adult", - "post_id": 508363 - }, - }), - ) + example = "https://fantia.jp/posts/12345" def __init__(self, match): FantiaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/fapachi.py b/gallery_dl/extractor/fapachi.py index ee6d15a1..6e815193 100644 --- a/gallery_dl/extractor/fapachi.py +++ b/gallery_dl/extractor/fapachi.py @@ -14,25 +14,13 @@ class FapachiPostExtractor(Extractor): """Extractor for individual posts on fapachi.com""" category = "fapachi" subcategory = "post" + root = "https://fapachi.com" directory_fmt = ("{category}", "{user}") filename_fmt = "{user}_{id}.{extension}" archive_fmt = "{user}_{id}" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search/)([^/?#]+)/media/(\d+)") - root = "https://fapachi.com" - test = ( - # NSFW - ("https://fapachi.com/sonson/media/0082", { - "pattern": (r"https://fapachi\.com/models/s/o/" - r"sonson/1/full/sonson_0082\.jpeg"), - "keyword": { - "user": "sonson", - "id" : "0082", - }, - }), - # NSFW - ("https://fapachi.com/ferxiita/media/0159"), - ) + example = "https://fapachi.com/MODEL/media/12345" def __init__(self, match): Extractor.__init__(self, match) @@ -54,17 +42,10 @@ class FapachiUserExtractor(Extractor): """Extractor for all posts from a fapachi user""" category = "fapachi" subcategory = "user" + root = "https://fapachi.com" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$") - root = "https://fapachi.com" - test = ( - ("https://fapachi.com/sonson", { - "pattern": FapachiPostExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://fapachi.com/ferxiita/page/3"), - ) + example = "https://fapachi.com/MODEL" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/fapello.py b/gallery_dl/extractor/fapello.py index d6fcb4b3..d4524e0e 100644 --- a/gallery_dl/extractor/fapello.py +++ b/gallery_dl/extractor/fapello.py @@ -19,32 +19,7 @@ class FapelloPostExtractor(Extractor): archive_fmt = "{type}_{model}_{id}" pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" r"/(?!search/|popular_videos/)([^/?#]+)/(\d+)") - test = ( - ("https://fapello.com/carrykey/530/", { - "pattern": (r"https://fapello\.com/content/c/a" - r"/carrykey/1000/carrykey_0530\.jpg"), - "keyword": { - "model": "carrykey", - "id" : 530, - "type" : "photo", - "thumbnail": "", - }, - }), - ("https://fapello.com/vladislava-661/693/", { - "pattern": (r"https://cdn\.fapello\.com/content/v/l" - r"/vladislava-661/1000/vladislava-661_0693\.mp4"), - "keyword": { - "model": "vladislava-661", - "id" : 693, - "type" : "video", - "thumbnail": ("https://fapello.com/content/v/l" - "/vladislava-661/1000/vladislava-661_0693.jpg"), - }, - }), - ("https://fapello.com/carrykey/000/", { - "exception": exception.NotFoundError, - }), - ) + example = "https://fapello.com/MODEL/12345/" def __init__(self, match): Extractor.__init__(self, match) @@ -77,14 +52,7 @@ class FapelloModelExtractor(Extractor): r"/(?!top-(?:likes|followers)|popular_videos" r"|videos|trending|search/?$)" r"([^/?#]+)/?$") - test = ( - ("https://fapello.com/hyoon/", { - "pattern": FapelloPostExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://fapello.com/kobaebeefboo/"), - ) + example = "https://fapello.com/model/" def __init__(self, match): Extractor.__init__(self, match) @@ -112,22 +80,7 @@ class FapelloPathExtractor(Extractor): pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" r"|popular_videos/[^/?#]+)/?$") - test = ( - ("https://fapello.com/top-likes/", { - "pattern": FapelloModelExtractor.pattern, - "range" : "1-10", - "count" : 10, - }), - ("https://fapello.com/videos/", { - "pattern": FapelloPostExtractor.pattern, - "range" : "1-10", - "count" : 10, - }), - ("https://fapello.com/top-followers/"), - ("https://fapello.com/trending/"), - ("https://fapello.com/popular_videos/twelve_hours/"), - ("https://fapello.com/popular_videos/week/"), - ) + example = "https://fapello.com/trending/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py index 3b18c63e..ea327654 100644 --- a/gallery_dl/extractor/flickr.py +++ b/gallery_dl/extractor/flickr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -64,42 +64,7 @@ class FlickrImageExtractor(FlickrExtractor): r"(?:(?:www\.|secure\.|m\.)?flickr\.com/photos/[^/?#]+/" r"|[\w-]+\.static\.?flickr\.com/(?:\d+/)+)(\d+)" r"|flic\.kr/p/([A-Za-z1-9]+))") - test = ( - ("https://www.flickr.com/photos/departingyyz/16089302239", { - "pattern": pattern, - "content": ("3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", - "0821a28ee46386e85b02b67cf2720063440a228c"), - "keyword": { - "comments": int, - "description": str, - "extension": "jpg", - "filename": "16089302239_de18cd8017_b", - "id": 16089302239, - "height": 683, - "label": "Large", - "media": "photo", - "url": str, - "views": int, - "width": 1024, - }, - }), - ("https://secure.flickr.com/photos/departingyyz/16089302239"), - ("https://m.flickr.com/photos/departingyyz/16089302239"), - ("https://flickr.com/photos/departingyyz/16089302239"), - - ("https://www.flickr.com/photos/145617051@N08/46733161535", { - "count": 1, - "keyword": {"media": "video"}, - }), - ("http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", { - "pattern": pattern}), - ("https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", { - "pattern": pattern}), - ("https://flic.kr/p/FPVo9U", { - "pattern": pattern}), - ("https://www.flickr.com/photos/zzz/16089302238", { - "exception": exception.NotFoundError}), - ) + example = "https://www.flickr.com/photos/USER/12345" def __init__(self, match): FlickrExtractor.__init__(self, match) @@ -145,18 +110,7 @@ class FlickrAlbumExtractor(FlickrExtractor): "Albums", "{album[id]} {album[title]}") archive_fmt = "a_{album[id]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/(?:album|set)s(?:/(\d+))?" - test = ( - (("https://www.flickr.com/photos/shona_s/albums/72157633471741607"), { - "pattern": FlickrImageExtractor.pattern, - "count": 6, - }), - ("https://www.flickr.com/photos/shona_s/albums", { - "pattern": pattern, - "count": 2, - }), - ("https://secure.flickr.com/photos/shona_s/albums"), - ("https://m.flickr.com/photos/shona_s/albums"), - ) + example = "https://www.flickr.com/photos/USER/albums/12345" def __init__(self, match): FlickrExtractor.__init__(self, match) @@ -194,11 +148,7 @@ class FlickrGalleryExtractor(FlickrExtractor): "Galleries", "{gallery[gallery_id]} {gallery[title]}") archive_fmt = "g_{gallery[id]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/galleries/(\d+)" - test = (("https://www.flickr.com/photos/flickr/" - "galleries/72157681572514792/"), { - "pattern": FlickrImageExtractor.pattern, - "count": ">= 10", - }) + example = "https://www.flickr.com/photos/USER/galleries/12345/" def __init__(self, match): FlickrExtractor.__init__(self, match) @@ -219,10 +169,7 @@ class FlickrGroupExtractor(FlickrExtractor): directory_fmt = ("{category}", "Groups", "{group[groupname]}") archive_fmt = "G_{group[nsid]}_{id}" pattern = BASE_PATTERN + r"/groups/([^/?#]+)" - test = ("https://www.flickr.com/groups/bird_headshots/", { - "pattern": FlickrImageExtractor.pattern, - "count": "> 150", - }) + example = "https://www.flickr.com/groups/NAME/" def metadata(self): self.group = self.api.urls_lookupGroup(self.item_id) @@ -237,10 +184,7 @@ class FlickrUserExtractor(FlickrExtractor): subcategory = "user" archive_fmt = "u_{user[nsid]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/?$" - test = ("https://www.flickr.com/photos/shona_s/", { - "pattern": FlickrImageExtractor.pattern, - "count": 28, - }) + example = "https://www.flickr.com/photos/USER/" def photos(self): return self.api.people_getPhotos(self.user["nsid"]) @@ -252,10 +196,7 @@ class FlickrFavoriteExtractor(FlickrExtractor): directory_fmt = ("{category}", "{user[username]}", "Favorites") archive_fmt = "f_{user[nsid]}_{id}" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/favorites" - test = ("https://www.flickr.com/photos/shona_s/favorites", { - "pattern": FlickrImageExtractor.pattern, - "count": 4, - }) + example = "https://www.flickr.com/photos/USER/favorites" def photos(self): return self.api.favorites_getList(self.user["nsid"]) @@ -267,11 +208,7 @@ class FlickrSearchExtractor(FlickrExtractor): directory_fmt = ("{category}", "Search", "{search[text]}") archive_fmt = "s_{search}_{id}" pattern = BASE_PATTERN + r"/search/?\?([^#]+)" - test = ( - ("https://flickr.com/search/?text=mountain"), - ("https://flickr.com/search/?text=tree%20cloud%20house" - "&color_codes=4&styles=minimalism"), - ) + example = "https://flickr.com/search/?text=QUERY" def __init__(self, match): FlickrExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index fefb2c4c..93ac5416 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -25,9 +25,6 @@ class FoolfuukaExtractor(BaseExtractor): if self.category == "b4k": self.remote = self._remote_direct - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): yield Message.Directory, self.metadata() for post in self.posts(): @@ -111,43 +108,7 @@ class FoolfuukaThreadExtractor(FoolfuukaExtractor): directory_fmt = ("{category}", "{board[shortname]}", "{thread_num} {title|comment[:50]}") pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)" - test = ( - ("https://archive.4plebs.org/tg/thread/54059290", { - "url": "fd823f17b5001442b941fddcd9ec91bafedfbc79", - }), - ("https://archived.moe/gd/thread/309639/", { - "url": "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", - "content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", - }), - ("https://archived.moe/a/thread/159767162/", { - "url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", - }), - ("https://archiveofsins.com/h/thread/4668813/", { - "url": "f612d287087e10a228ef69517cf811539db9a102", - "content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", - }), - ("https://arch.b4k.co/meta/thread/196/", { - "url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", - }), - ("https://desuarchive.org/a/thread/159542679/", { - "url": "e7d624aded15a069194e38dc731ec23217a422fb", - }), - ("https://boards.fireden.net/sci/thread/11264294/", { - "url": "61cab625c95584a12a30049d054931d64f8d20aa", - }), - ("https://archive.palanq.win/c/thread/4209598/", { - "url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", - }), - ("https://rbt.asia/g/thread/61487650/", { - "url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", - }), - ("https://archive.rebeccablacktech.com/g/thread/61487650/", { - "url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", - }), - ("https://thebarchive.com/b/thread/739772332/", { - "url": "e8b18001307d130d67db31740ce57c8561b5d80c", - }), - ) + example = "https://archived.moe/a/thread/12345/" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -175,17 +136,7 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka based boards/archives""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)/\d*$" - test = ( - ("https://archive.4plebs.org/tg/"), - ("https://archived.moe/gd/"), - ("https://archiveofsins.com/h/"), - ("https://arch.b4k.co/meta/"), - ("https://desuarchive.org/a/"), - ("https://boards.fireden.net/sci/"), - ("https://archive.palanq.win/c/"), - ("https://rbt.asia/g/"), - ("https://thebarchive.com/b/"), - ) + example = "https://archived.moe/a/" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -217,18 +168,8 @@ class FoolfuukaSearchExtractor(FoolfuukaExtractor): subcategory = "search" directory_fmt = ("{category}", "search", "{search}") pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)" + example = "https://archived.moe/_/search/text/QUERY/" request_interval = 1.0 - test = ( - ("https://archive.4plebs.org/_/search/text/test/"), - ("https://archived.moe/_/search/text/test/"), - ("https://archiveofsins.com/_/search/text/test/"), - ("https://archiveofsins.com/_/search/text/test/"), - ("https://desuarchive.org/_/search/text/test/"), - ("https://boards.fireden.net/_/search/text/test/"), - ("https://archive.palanq.win/_/search/text/test/"), - ("https://rbt.asia/_/search/text/test/"), - ("https://thebarchive.com/_/search/text/test/"), - ) def __init__(self, match): FoolfuukaExtractor.__init__(self, match) @@ -283,17 +224,7 @@ class FoolfuukaGalleryExtractor(FoolfuukaExtractor): subcategory = "gallery" directory_fmt = ("{category}", "{board}", "gallery") pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?" - test = ( - ("https://archive.4plebs.org/tg/gallery/1"), - ("https://archived.moe/gd/gallery/2"), - ("https://archiveofsins.com/h/gallery/3"), - ("https://arch.b4k.co/meta/gallery/"), - ("https://desuarchive.org/a/gallery/5"), - ("https://boards.fireden.net/sci/gallery/6"), - ("https://archive.palanq.win/c/gallery"), - ("https://rbt.asia/g/gallery/8"), - ("https://thebarchive.com/b/gallery/9"), - ) + example = "https://archived.moe/a/gallery" def __init__(self, match): FoolfuukaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 57d37b76..b0699b03 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -53,13 +53,7 @@ class FoolslideChapterExtractor(FoolslideExtractor): "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" - test = ( - (("https://read.powermanga.org" - "/read/one_piece_digital_colour_comics/en/0/75/"), { - "url": "854c5817f8f767e1bccd05fa9d58ffb5a4b09384", - "keyword": "a60c42f2634b7387899299d411ff494ed0ad6dbe", - }), - ) + example = "https://read.powermanga.org/read/MANGA/en/0/123/" def items(self): page = self.request(self.gallery_url).text @@ -103,23 +97,7 @@ class FoolslideMangaExtractor(FoolslideExtractor): subcategory = "manga" categorytransfer = True pattern = BASE_PATTERN + r"(/series/[^/?#]+)" - test = ( - (("https://read.powermanga.org" - "/series/one_piece_digital_colour_comics/"), { - "count": ">= 1", - "keyword": { - "chapter": int, - "chapter_minor": str, - "chapter_string": str, - "group": "PowerManga", - "lang": "en", - "language": "English", - "manga": "One Piece Digital Colour Comics", - "title": str, - "volume": int, - }, - }), - ) + example = "https://read.powermanga.org/series/MANGA/" def items(self): page = self.request(self.gallery_url).text diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 8c3ef79d..7dc64193 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -31,6 +31,7 @@ class FuraffinityExtractor(Extractor): def _init(self): self.offset = 0 + self.external = self.config("external", False) if self.config("descriptions") == "html": self._process_description = str.strip @@ -41,13 +42,12 @@ class FuraffinityExtractor(Extractor): else: self._new_layout = None - def items(self): if self._warning: if not self.cookies_check(self.cookies_names): self.log.warning("no 'a' and 'b' session cookies set") FuraffinityExtractor._warning = False - external = self.config("external", False) + def items(self): metadata = self.metadata() for post_id in util.advance(self.posts(), self.offset): post = self._parse_post(post_id) @@ -57,7 +57,7 @@ class FuraffinityExtractor(Extractor): yield Message.Directory, post yield Message.Url, post["url"], post - if external: + if self.external: for url in text.extract_iter( post["_description"], 'href="http', '"'): yield Message.Queue, "http" + url, post @@ -219,12 +219,7 @@ class FuraffinityGalleryExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's gallery""" subcategory = "gallery" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)" - test = ("https://www.furaffinity.net/gallery/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "range": "45-50", - "count": 6, - }) + example = "https://www.furaffinity.net/gallery/USER/" def posts(self): return self._pagination("gallery") @@ -235,11 +230,7 @@ class FuraffinityScrapsExtractor(FuraffinityExtractor): subcategory = "scraps" directory_fmt = ("{category}", "{user!l}", "Scraps") pattern = BASE_PATTERN + r"/scraps/([^/?#]+)" - test = ("https://www.furaffinity.net/scraps/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+(/stories)?/\d+/\d+.\w+.", - "count": ">= 3", - }) + example = "https://www.furaffinity.net/scraps/USER/" def posts(self): return self._pagination("scraps") @@ -250,13 +241,7 @@ class FuraffinityFavoriteExtractor(FuraffinityExtractor): subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "Favorites") pattern = BASE_PATTERN + r"/favorites/([^/?#]+)" - test = ("https://www.furaffinity.net/favorites/mirlinthloth/", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+/\d+/\d+.\w+\.\w+", - "keyword": {"favorite_id": int}, - "range": "45-50", - "count": 6, - }) + example = "https://www.furaffinity.net/favorites/USER/" def posts(self): return self._pagination_favorites() @@ -273,19 +258,7 @@ class FuraffinitySearchExtractor(FuraffinityExtractor): subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)" - test = ( - ("https://www.furaffinity.net/search/?q=cute", { - "pattern": r"https://d\d?\.f(uraffinity|acdn)\.net" - r"/art/[^/]+/\d+/\d+.\w+\.\w+", - "range": "45-50", - "count": 6, - }), - # first page of search results (#2402) - ("https://www.furaffinity.net/search/?q=leaf&range=1day", { - "range": "1-3", - "count": 3, - }), - ) + example = "https://www.furaffinity.net/search/?q=QUERY" def __init__(self, match): FuraffinityExtractor.__init__(self, match) @@ -304,65 +277,7 @@ class FuraffinityPostExtractor(FuraffinityExtractor): """Extractor for individual posts on furaffinity""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)" - test = ( - ("https://www.furaffinity.net/view/21835115/", { - "pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art" - r"/mirlinthloth/music/1488278723/1480267446.mirlinthlot" - r"h_dj_fennmink_-_bude_s_4_ever\.mp3", - "keyword": { - "artist" : "mirlinthloth", - "artist_url" : "mirlinthloth", - "date" : "dt:2016-11-27 17:24:06", - "description": "A Song made playing the game Cosmic DJ.", - "extension" : "mp3", - "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", - "id" : 21835115, - "tags" : list, - "title" : "Bude's 4 Ever", - "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", - "user" : "mirlinthloth", - "views" : int, - "favorites" : int, - "comments" : int, - "rating" : "General", - "fa_category": "Music", - "theme" : "All", - "species" : "Unspecified / Any", - "gender" : "Any", - "width" : 120, - "height" : 120, - }, - }), - # 'external' option (#1492) - ("https://www.furaffinity.net/view/42166511/", { - "options": (("external", True),), - "pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/" - r"|http://www\.postybirb\.com", - "count": 2, - }), - # no tags (#2277) - ("https://www.furaffinity.net/view/45331225/", { - "keyword": { - "artist": "Kota_Remminders", - "artist_url": "kotaremminders", - "date": "dt:2022-01-03 17:49:33", - "fa_category": "Adoptables", - "filename": "1641232173.kotaremminders_chidopts1", - "gender": "Any", - "height": 905, - "id": 45331225, - "rating": "General", - "species": "Unspecified / Any", - "tags": [], - "theme": "All", - "title": "REMINDER", - "width": 1280, - }, - }), - ("https://furaffinity.net/view/21835115/"), - ("https://sfw.furaffinity.net/view/21835115/"), - ("https://www.furaffinity.net/full/21835115/"), - ) + example = "https://www.furaffinity.net/view/12345/" def posts(self): post_id = self.user @@ -375,16 +290,7 @@ class FuraffinityUserExtractor(FuraffinityExtractor): subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/user/([^/?#]+)" - test = ( - ("https://www.furaffinity.net/user/mirlinthloth/", { - "pattern": r"/gallery/mirlinthloth/$", - }), - ("https://www.furaffinity.net/user/mirlinthloth/", { - "options": (("include", "all"),), - "pattern": r"/(gallery|scraps|favorites)/mirlinthloth/$", - "count": 3, - }), - ) + example = "https://www.furaffinity.net/user/USER/" def initialize(self): pass @@ -402,11 +308,7 @@ class FuraffinityFollowingExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's watched users""" subcategory = "following" pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)" - test = ("https://www.furaffinity.net/watchlist/by/mirlinthloth/", { - "pattern": FuraffinityUserExtractor.pattern, - "range": "176-225", - "count": 50, - }) + example = "https://www.furaffinity.net/watchlist/by/USER/" def items(self): url = "{}/watchlist/by/{}/".format(self.root, self.user) diff --git a/gallery_dl/extractor/fuskator.py b/gallery_dl/extractor/fuskator.py index d6640f57..20afb5a4 100644 --- a/gallery_dl/extractor/fuskator.py +++ b/gallery_dl/extractor/fuskator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,22 +18,7 @@ class FuskatorGalleryExtractor(GalleryExtractor): category = "fuskator" root = "https://fuskator.com" pattern = r"(?:https?://)?fuskator\.com/(?:thumbs|expanded)/([^/?#]+)" - test = ( - ("https://fuskator.com/thumbs/d0GnIzXrSKU/", { - "pattern": r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", - "count": 22, - "keyword": { - "gallery_id": 473023, - "gallery_hash": "d0GnIzXrSKU", - "title": "re:Shaved Brunette Babe Maria Ryabushkina with ", - "views": int, - "score": float, - "count": 22, - "tags": list, - }, - }), - ("https://fuskator.com/expanded/gXpKzjgIidA/index.html"), - ) + example = "https://fuskator.com/thumbs/ID/" def __init__(self, match): self.gallery_hash = match.group(1) @@ -82,13 +67,7 @@ class FuskatorSearchExtractor(Extractor): subcategory = "search" root = "https://fuskator.com" pattern = r"(?:https?://)?fuskator\.com(/(?:search|page)/.+)" - test = ( - ("https://fuskator.com/search/red_swimsuit/", { - "pattern": FuskatorGalleryExtractor.pattern, - "count": ">= 40", - }), - ("https://fuskator.com/page/3/swimsuit/quality/"), - ) + example = "https://fuskator.com/search/TAG/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index e2173de9..b62ff78b 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -115,18 +115,7 @@ class GelbooruTagExtractor(GelbooruBase, gelbooru_v02.GelbooruV02TagExtractor): """Extractor for images from gelbooru.com based on search-tags""" pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]+)" - test = ( - ("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", { - "count": 5, - }), - ("https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", { - "range": "196-204", - "url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", - "pattern": r"https://img\d\.gelbooru\.com" - r"/images/../../[0-9a-f]{32}\.jpg", - "count": 9, - }), - ) + example = "https://gelbooru.com/index.php?page=post&s=list&tags=TAG" class GelbooruPoolExtractor(GelbooruBase, @@ -134,11 +123,7 @@ class GelbooruPoolExtractor(GelbooruBase, """Extractor for gelbooru pools""" per_page = 45 pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)" - test = ( - ("https://gelbooru.com/index.php?page=pool&s=show&id=761", { - "count": 6, - }), - ) + example = "https://gelbooru.com/index.php?page=pool&s=show&id=12345" skip = GelbooruBase._skip_offset @@ -169,9 +154,7 @@ class GelbooruFavoriteExtractor(GelbooruBase, """Extractor for gelbooru favorites""" per_page = 100 pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" - test = ("https://gelbooru.com/index.php?page=favorites&s=view&id=279415", { - "count": 3, - }) + example = "https://gelbooru.com/index.php?page=favorites&s=view&id=12345" skip = GelbooruBase._skip_offset @@ -221,76 +204,21 @@ class GelbooruPostExtractor(GelbooruBase, r"(?=(?:[^#]+&)?page=post(?:&|#|$))" r"(?=(?:[^#]+&)?s=view(?:&|#|$))" r"(?:[^#]+&)?id=(\d+)") - test = ( - ("https://gelbooru.com/index.php?page=post&s=view&id=313638", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "count": 1, - }), - - ("https://gelbooru.com/index.php?page=post&s=view&id=313638"), - ("https://gelbooru.com/index.php?s=view&page=post&id=313638"), - ("https://gelbooru.com/index.php?page=post&id=313638&s=view"), - ("https://gelbooru.com/index.php?s=view&id=313638&page=post"), - ("https://gelbooru.com/index.php?id=313638&page=post&s=view"), - ("https://gelbooru.com/index.php?id=313638&s=view&page=post"), - - ("https://gelbooru.com/index.php?page=post&s=view&id=6018318", { - "options": (("tags", True),), - "content": "977caf22f27c72a5d07ea4d4d9719acdab810991", - "keyword": { - "tags_artist": "kirisaki_shuusei", - "tags_character": str, - "tags_copyright": "vocaloid", - "tags_general": str, - "tags_metadata": str, - }, - }), - # video - ("https://gelbooru.com/index.php?page=post&s=view&id=5938076", { - "content": "6360452fa8c2f0c1137749e81471238564df832a", - "pattern": r"https://img\d\.gelbooru\.com/images" - r"/22/61/226111273615049235b001b381707bd0\.webm", - }), - # notes - ("https://gelbooru.com/index.php?page=post&s=view&id=5997331", { - "options": (("notes", True),), - "keyword": { - "notes": [ - { - "body": "Look over this way when you talk~", - "height": 553, - "width": 246, - "x": 35, - "y": 72, - }, - { - "body": "Hey~\nAre you listening~?", - "height": 557, - "width": 246, - "x": 1233, - "y": 109, - }, - ], - }, - }), - ) + example = "https://gelbooru.com/index.php?page=post&s=view&id=12345" class GelbooruRedirectExtractor(GelbooruBase, Extractor): subcategory = "redirect" pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com" r"/redirect\.php\?s=([^&#]+)") - test = (("https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgu" - "cGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng=="), { - "pattern": r"https://gelbooru.com/index.php" - r"\?page=post&s=view&id=1830046" - }) + example = "https://gelbooru.com/redirect.php?s=BASE64" def __init__(self, match): Extractor.__init__(self, match) - self.redirect_url = text.ensure_http_scheme( - binascii.a2b_base64(match.group(1)).decode()) + self.url_base64 = match.group(1) def items(self): + url = text.ensure_http_scheme(binascii.a2b_base64( + self.url_base64).decode()) data = {"_extractor": GelbooruPostExtractor} - yield Message.Queue, self.redirect_url, data + yield Message.Queue, url, data diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index b6fbcb6d..0b96048b 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -90,24 +90,7 @@ class GelbooruV01TagExtractor(GelbooruV01Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=post&s=list&tags=parody"), { - "range": "1-25", - "count": 25, - }), - (("https://illusioncards.booru.org" - "/index.php?page=post&s=list&tags=koikatsu"), { - "range": "1-25", - "count": 25, - }), - ("https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", { - "range": "1-25", - "count": 25, - }), - ("https://drawfriends.booru.org/index.php?page=post&s=list&tags=all"), - ("https://vidyart2.booru.org/index.php?page=post&s=list&tags=all"), - ) + example = "https://allgirl.booru.org/index.php?page=post&s=list&tags=TAG" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) @@ -128,21 +111,7 @@ class GelbooruV01FavoriteExtractor(GelbooruV01Extractor): archive_fmt = "f_{favorite_id}_{id}" per_page = 50 pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=favorites&s=view&id=1166"), { - "count": 2, - }), - (("https://illusioncards.booru.org" - "/index.php?page=favorites&s=view&id=84887"), { - "count": 2, - }), - ("https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", { - "count": 4, - }), - ("https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1"), - ("https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1"), - ) + example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=1" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) @@ -161,40 +130,7 @@ class GelbooruV01PostExtractor(GelbooruV01Extractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" - test = ( - (("https://the-collection.booru.org" - "/index.php?page=post&s=view&id=100520"), { - "url": "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", - "content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", - }), - (("https://illusioncards.booru.org" - "/index.php?page=post&s=view&id=82746"), { - "url": "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", - "content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", - }), - ("https://allgirl.booru.org/index.php?page=post&s=view&id=107213", { - "url": "b416800d2d2b072f80d3b37cfca9cb806fb25d51", - "content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", - "keyword": { - "created_at": "2021-02-13 16:27:39", - "date": "dt:2021-02-13 16:27:39", - "file_url": "https://img.booru.org/allgirl//images/107" - "/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", - "height": "1200", - "id": "107213", - "md5": "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", - "rating": "s", - "score": str, - "source": "", - "tags": "blush dress green_eyes green_hair hatsune_miku " - "long_hair twintails vocaloid", - "uploader": "Honochi31", - "width": "1600" - }, - }), - ("https://drawfriends.booru.org/index.php?page=post&s=view&id=107474"), - ("https://vidyart2.booru.org/index.php?page=post&s=view&id=39168"), - ) + example = "https://allgirl.booru.org/index.php?page=post&s=view&id=12345" def __init__(self, match): GelbooruV01Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index 1ef78efd..0864b9f6 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -183,6 +183,10 @@ INSTANCES = { "root": "https://hypnohub.net", "pattern": r"hypnohub\.net", }, + "xbooru": { + "root": "https://xbooru.com", + "pattern": r"xbooru\.com", + }, } BASE_PATTERN = GelbooruV02Extractor.update(INSTANCES) @@ -193,27 +197,7 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" - test = ( - ("https://rule34.xxx/index.php?page=post&s=list&tags=danraku", { - "content": ("5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", - "622e80be3f496672c44aab5c47fbc6941c61bc79"), - "pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", - "count": 2, - }), - ("https://safebooru.org/index.php?page=post&s=list&tags=bonocho", { - "url": "17c61b386530cf4c30842c9f580d15ef1cd09586", - "content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", - }), - ("https://realbooru.com/index.php?page=post&s=list&tags=wine", { - "count": ">= 64", - }), - ("https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", { - "count": ">= 120", - }), - ("https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", { - "url": "fe662b86d38c331fcac9c62af100167d404937dc", - }), - ) + example = "https://safebooru.org/index.php?page=post&s=list&tags=TAG" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -232,21 +216,7 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/index\.php\?page=pool&s=show&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=pool&s=show&id=179", { - "count": 3, - }), - ("https://safebooru.org/index.php?page=pool&s=show&id=11", { - "count": 5, - }), - ("https://realbooru.com/index.php?page=pool&s=show&id=1", { - "count": 3, - }), - ("https://hypnohub.net/index.php?page=pool&s=show&id=61", { - "url": "d314826280073441a2da609f70ee814d1f4b9407", - "count": 3, - }), - ) + example = "https://safebooru.org/index.php?page=pool&s=show&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -298,23 +268,7 @@ class GelbooruV02FavoriteExtractor(GelbooruV02Extractor): archive_fmt = "f_{favorite_id}_{id}" per_page = 50 pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", { - "count": 3, - }), - ("https://safebooru.org/index.php?page=favorites&s=view&id=17567", { - "count": 2, - }), - ("https://realbooru.com/index.php?page=favorites&s=view&id=274", { - "count": 2, - }), - ("https://tbib.org/index.php?page=favorites&s=view&id=7881", { - "count": 3, - }), - ("https://hypnohub.net/index.php?page=favorites&s=view&id=43546", { - "count": 3, - }), - ) + example = "https://safebooru.org/index.php?page=favorites&s=view&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) @@ -335,112 +289,7 @@ class GelbooruV02PostExtractor(GelbooruV02Extractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" - test = ( - ("https://rule34.xxx/index.php?page=post&s=view&id=863", { - "pattern": r"https://api-cdn\.rule34\.xxx/images" - r"/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", - "content": ("a43f418aa350039af0d11cae501396a33bbe2201", - "67b516295950867e1c1ab6bc13b35d3b762ed2a3"), - "options": (("tags", True), ("notes", True)), - "keyword": { - "tags_artist": "reverse_noise yamu_(reverse_noise)", - "tags_character": "hong_meiling", - "tags_copyright": "touhou", - "tags_general": str, - "tags_metadata": "censored translated", - "notes": [ - { - "body": "It feels angry, I'm losing myself... " - "It won't calm down!", - "height": 65, - "id": 93586, - "width": 116, - "x": 22, - "y": 333, - }, - { - "body": "REPUTATION OF RAGE", - "height": 272, - "id": 93587, - "width": 199, - "x": 78, - "y": 442, - }, - ], - - }, - }), - ("https://hypnohub.net/index.php?page=post&s=view&id=1439", { - "pattern": r"https://hypnohub\.net/images" - r"/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", - "content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", - "options": (("tags", True), ("notes", True)), - "keyword": { - "tags_artist": "brokenteapot", - "tags_character": "hsien-ko", - "tags_copyright": "capcom darkstalkers", - "tags_general": str, - "tags_metadata": "dialogue text translated", - "notes": [ - { - "body": "Master Master Master " - "Master Master Master", - "height": 83, - "id": 10577, - "width": 129, - "x": 259, - "y": 20, - }, - { - "body": "Response Response Response " - "Response Response Response", - "height": 86, - "id": 10578, - "width": 125, - "x": 126, - "y": 20, - }, - { - "body": "Obedience Obedience Obedience " - "Obedience Obedience Obedience", - "height": 80, - "id": 10579, - "width": 98, - "x": 20, - "y": 20, - }, - ], - - }, - }), - ("https://safebooru.org/index.php?page=post&s=view&id=1169132", { - "url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b", - "content": "93b293b27dabd198afafabbaf87c49863ac82f27", - "options": (("tags", True),), - "keyword": { - "tags_artist": "kawanakajima", - "tags_character": "heath_ledger ronald_mcdonald the_joker", - "tags_copyright": "dc_comics mcdonald's the_dark_knight", - "tags_general": str, - }, - }), - ("https://realbooru.com/index.php?page=post&s=view&id=668483", { - "pattern": r"https://realbooru\.com//?images/dc/b5" - r"/dcb5c0ce9ec0bf74a6930608985f4719\.jpeg", - "content": "7f5873ce3b6cd295ea2e81fcb49583098ea9c8da", - "options": (("tags", True),), - "keyword": { - "tags_general": "1girl blonde blonde_hair blue_eyes cute " - "female female_only looking_at_viewer smile " - "solo solo_female teeth", - "tags_model": "jennifer_lawrence", - }, - }), - ("https://tbib.org/index.php?page=post&s=view&id=9233957", { - "url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", - "content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", - }), - ) + example = "https://safebooru.org/index.php?page=post&s=view&id=12345" def __init__(self, match): GelbooruV02Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/generic.py b/gallery_dl/extractor/generic.py index 4ab26ae7..16d4340c 100644 --- a/gallery_dl/extractor/generic.py +++ b/gallery_dl/extractor/generic.py @@ -34,31 +34,7 @@ class GenericExtractor(Extractor): r"(?:\?(?P[^#]*))?" # optional query r"(?:\#(?P.*))?" # optional fragment ) - - test = ( - ("generic:https://www.nongnu.org/lzip/", { - "count": 1, - "content": "40be5c77773d3e91db6e1c5df720ee30afb62368", - "keyword": { - "description": "Lossless data compressor", - "imageurl": "https://www.nongnu.org/lzip/lzip.png", - "keywords": "lzip, clzip, plzip, lzlib, LZMA, bzip2, " - "gzip, data compression, GNU, free software", - "pageurl": "https://www.nongnu.org/lzip/", - }, - }), - # internationalized domain name - ("generic:https://räksmörgås.josefsson.org/", { - "count": 2, - "pattern": "^https://räksmörgås.josefsson.org/", - }), - ("g:https://en.wikipedia.org/Main_Page"), - ("g:https://example.org/path/to/file?que=1?&ry=2/#fragment"), - ("g:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"), - ("generic:https://en.wikipedia.org/Main_Page"), - ("generic:https://example.org/path/to/file?que=1?&ry=2/#fragment"), - ("generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"), - ) + example = "generic:https://www.nongnu.org/lzip/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/gfycat.py b/gallery_dl/extractor/gfycat.py deleted file mode 100644 index 53ef1180..00000000 --- a/gallery_dl/extractor/gfycat.py +++ /dev/null @@ -1,306 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2017-2023 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Extractors for https://gfycat.com/""" - -from .common import Extractor, Message -from .. import text, exception -from ..cache import cache - - -class GfycatExtractor(Extractor): - """Base class for gfycat extractors""" - category = "gfycat" - filename_fmt = "{category}_{gfyName}{title:?_//}.{extension}" - archive_fmt = "{gfyName}" - root = "https://gfycat.com" - - def __init__(self, match): - Extractor.__init__(self, match) - self.key = match.group(1).lower() - - def _init(self): - formats = self.config("format") - if formats is None: - formats = ("mp4", "webm", "mobile", "gif") - elif isinstance(formats, str): - formats = (formats, "mp4", "webm", "mobile", "gif") - self.formats = formats - - def items(self): - metadata = self.metadata() - for gfycat in self.gfycats(): - if "gfyName" not in gfycat: - self.log.warning("Skipping '%s' (malformed)", gfycat["gfyId"]) - continue - - url = self._process(gfycat) - if not url: - self.log.warning("Skipping '%s' (format not available)", - gfycat["gfyId"]) - continue - - gfycat.update(metadata) - yield Message.Directory, gfycat - yield Message.Url, url, gfycat - - def _process(self, gfycat): - gfycat["_fallback"] = formats = self._formats(gfycat) - gfycat["date"] = text.parse_timestamp(gfycat.get("createDate")) - return next(formats, None) - - def _formats(self, gfycat): - for fmt in self.formats: - key = fmt + "Url" - if key in gfycat: - url = gfycat[key] - if url.startswith("http:"): - url = "https" + url[4:] - gfycat["extension"] = url.rpartition(".")[2] - yield url - - def metadata(self): - return {} - - def gfycats(self): - return () - - -class GfycatUserExtractor(GfycatExtractor): - """Extractor for gfycat user profiles""" - subcategory = "user" - directory_fmt = ("{category}", "{username}") - pattern = r"(?:https?://)?gfycat\.com/@([^/?#]+)/?(?:$|\?|#)" - test = ("https://gfycat.com/@gretta", { - "pattern": r"https://giant\.gfycat\.com/[A-Za-z]+\.mp4", - "count": ">= 100", - }) - - def gfycats(self): - if self.key == "me": - return GfycatAPI(self).me() - return GfycatAPI(self).user(self.key) - - -class GfycatCollectionExtractor(GfycatExtractor): - """Extractor for a gfycat collection""" - subcategory = "collection" - directory_fmt = ("{category}", "{collection_owner}", - "{collection_name|collection_id}") - pattern = (r"(?:https?://)?gfycat\.com/@([^/?#]+)/collections" - r"/(\w+)(?:/([^/?#]+))?") - test = ("https://gfycat.com/@reactions/collections/nHgy2DtE/no-text", { - "pattern": r"https://\w+\.gfycat\.com/[A-Za-z]+\.mp4", - "count": ">= 100", - }) - - def __init__(self, match): - GfycatExtractor.__init__(self, match) - self.collection_id = match.group(2) - self.collection_name = match.group(3) - - def metadata(self): - return { - "collection_owner": self.key, - "collection_name" : self.collection_name, - "collection_id" : self.collection_id, - } - - def gfycats(self): - return GfycatAPI(self).collection(self.key, self.collection_id) - - -class GfycatCollectionsExtractor(GfycatExtractor): - """Extractor for a gfycat user's collections""" - subcategory = "collections" - pattern = r"(?:https?://)?gfycat\.com/@([^/?#]+)/collections/?(?:$|\?|#)" - test = ("https://gfycat.com/@sannahparker/collections", { - "pattern": GfycatCollectionExtractor.pattern, - "count": ">= 20", - }) - - def items(self): - for col in GfycatAPI(self).collections(self.key): - url = "https://gfycat.com/@{}/collections/{}/{}".format( - col["userId"], col["folderId"], col["linkText"]) - col["_extractor"] = GfycatCollectionExtractor - yield Message.Queue, url, col - - -class GfycatSearchExtractor(GfycatExtractor): - """Extractor for gfycat search results""" - subcategory = "search" - directory_fmt = ("{category}", "Search", "{search}") - pattern = r"(?:https?://)?gfycat\.com/gifs/search/([^/?#]+)" - test = ("https://gfycat.com/gifs/search/funny+animals", { - "pattern": r"https://\w+\.gfycat\.com/[A-Za-z]+\.mp4", - "archive": False, - "range": "100-300", - "count": "> 200", - }) - - def metadata(self): - self.key = text.unquote(self.key).replace("+", " ") - return {"search": self.key} - - def gfycats(self): - return GfycatAPI(self).search(self.key) - - -class GfycatImageExtractor(GfycatExtractor): - """Extractor for individual images from gfycat.com""" - subcategory = "image" - pattern = (r"(?:https?://)?(?:\w+\.)?gfycat\.com" - r"/(?:gifs/detail/|\w+/)?([A-Za-z]{8,})") - test = ( - ("https://gfycat.com/GrayGenerousCowrie", { - "url": "e0b5e1d7223108249b15c3c7898dd358dbfae045", - "content": "5786028e04b155baa20b87c5f4f77453cd5edc37", - "keyword": { - "gfyId": "graygenerouscowrie", - "gfyName": "GrayGenerousCowrie", - "gfyNumber": 755075459, - "title": "Bottom's up", - "username": "jackson3oh3", - "createDate": 1495884169, - "date": "dt:2017-05-27 11:22:49", - "md5": "a4796e05b0db9ba9ce5140145cd318aa", - "width": 400, - "height": 224, - "frameRate": 23.0, - "numFrames": 158.0, - "views": int, - }, - }), - (("https://thumbs.gfycat.com/SillyLameIsabellinewheatear" - "-size_restricted.gif"), { - "url": "13b32e6cc169d086577d7dd3fd36ee6cdbc02726", - }), - ("https://gfycat.com/detail/UnequaledHastyAnkole?tagname=aww", { - "url": "e24c9f69897fd223343782425a429c5cab6a768e", - }), - # retry 404'ed videos on redgifs (#874) - ("https://www.gfycat.com/foolishforkedabyssiniancat", { - "pattern": "https://redgifs.com/watch/foolishforkedabyssiniancat", - }), - # malformed API response (#902) - ("https://gfycat.com/illexcitablehairstreak", { - "count": 0, - }), - ("https://gfycat.com/gifs/detail/UnequaledHastyAnkole"), - ("https://gfycat.com/ifr/UnequaledHastyAnkole"), - ("https://gfycat.com/ru/UnequaledHastyAnkole"), - ) - - def items(self): - try: - gfycat = GfycatAPI(self).gfycat(self.key) - except exception.HttpError: - from .redgifs import RedgifsImageExtractor - url = "https://redgifs.com/watch/" + self.key - data = {"_extractor": RedgifsImageExtractor} - yield Message.Queue, url, data - else: - if "gfyName" not in gfycat: - self.log.warning("Skipping '%s' (malformed)", gfycat["gfyId"]) - return - url = self._process(gfycat) - if not url: - self.log.warning("Skipping '%s' (format not available)", - gfycat["gfyId"]) - return - yield Message.Directory, gfycat - yield Message.Url, url, gfycat - - -class GfycatAPI(): - API_ROOT = "https://api.gfycat.com" - - def __init__(self, extractor): - self.extractor = extractor - self.headers = {} - self.username, self.password = extractor._get_auth_info() - - def collection(self, user, collection): - endpoint = "/v1/users/{}/collections/{}/gfycats".format( - user, collection) - params = {"count": 100} - return self._pagination(endpoint, params) - - def collections(self, user): - endpoint = "/v1/users/{}/collections".format(user) - params = {"count": 100} - return self._pagination(endpoint, params, "gfyCollections") - - def gfycat(self, gfycat_id): - endpoint = "/v1/gfycats/" + gfycat_id - return self._call(endpoint)["gfyItem"] - - def me(self): - endpoint = "/v1/me/gfycats" - params = {"count": 100} - return self._pagination(endpoint, params) - - def search(self, query): - endpoint = "/v1/gfycats/search" - params = {"search_text": query, "count": 150} - return self._pagination(endpoint, params) - - def user(self, user): - endpoint = "/v1/users/{}/gfycats".format(user.lower()) - params = {"count": 100} - return self._pagination(endpoint, params) - - def authenticate(self): - self.headers["Authorization"] = \ - self._authenticate_impl(self.username, self.password) - - @cache(maxage=3600, keyarg=1) - def _authenticate_impl(self, username, password): - self.extractor.log.info("Logging in as %s", username) - - url = "https://weblogin.gfycat.com/oauth/webtoken" - headers = {"Origin": "https://gfycat.com"} - data = { - "access_key": "Anr96uuqt9EdamSCwK4txKPjMsf2" - "M95Rfa5FLLhPFucu8H5HTzeutyAa", - } - response = self.extractor.request( - url, method="POST", headers=headers, json=data).json() - - url = "https://weblogin.gfycat.com/oauth/weblogin" - headers["authorization"] = "Bearer " + response["access_token"] - data = { - "grant_type": "password", - "username" : username, - "password" : password, - } - response = self.extractor.request( - url, method="POST", headers=headers, json=data, fatal=None).json() - - if "errorMessage" in response: - raise exception.AuthenticationError( - response["errorMessage"]["description"]) - return "Bearer " + response["access_token"] - - def _call(self, endpoint, params=None): - if self.username: - self.authenticate() - - url = self.API_ROOT + endpoint - return self.extractor.request( - url, params=params, headers=self.headers).json() - - def _pagination(self, endpoint, params, key="gfycats"): - while True: - data = self._call(endpoint, params) - yield from data[key] - - if not data["cursor"]: - return - params["cursor"] = data["cursor"] diff --git a/gallery_dl/extractor/gofile.py b/gallery_dl/extractor/gofile.py index 60886a9d..3928792e 100644 --- a/gallery_dl/extractor/gofile.py +++ b/gallery_dl/extractor/gofile.py @@ -4,6 +4,8 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +"""Extractors for https://gofile.io/""" + from .common import Extractor, Message from .. import text, exception from ..cache import cache, memcache @@ -17,49 +19,7 @@ class GofileFolderExtractor(Extractor): directory_fmt = ("{category}", "{name} ({code})") archive_fmt = "{id}" pattern = r"(?:https?://)?(?:www\.)?gofile\.io/d/([^/?#]+)" - test = ( - ("https://gofile.io/d/k6BomI", { - "pattern": r"https://store\d+\.gofile\.io/download" - r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}" - r"/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", - "keyword": { - "createTime": int, - "directLink": "re:https://store5.gofile.io/download/direct/.+", - "downloadCount": int, - "extension": "png", - "filename": "test-テスト-%22&!", - "folder": { - "childs": [ - "b0367d79-b8ba-407f-8342-aaf8eb815443", - "7fd4a36a-c1dd-49ff-9223-d93f7d24093f" - ], - "code": "k6BomI", - "createTime": 1654076165, - "id": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "name": "root", - "public": True, - "totalDownloadCount": int, - "totalSize": 182, - "type": "folder" - }, - "id": r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", - "link": r"re:https://store5.gofile.io/download/.+\.png", - "md5": "re:[0-9a-f]{32}", - "mimetype": "image/png", - "name": "test-テスト-%22&!.png", - "num": int, - "parentFolder": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "serverChoosen": "store5", - "size": 182, - "thumbnail": r"re:https://store5.gofile.io/download/.+\.png", - "type": "file" - }, - }), - ("https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", { - "options": (("website-token", None),), - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ) + example = "https://gofile.io/d/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py index 5b561ead..a5221409 100644 --- a/gallery_dl/extractor/hbrowse.py +++ b/gallery_dl/extractor/hbrowse.py @@ -47,11 +47,7 @@ class HbrowseChapterExtractor(HbrowseBase, ChapterExtractor): "{page:>03}.{extension}") archive_fmt = "{manga_id}_{chapter}_{page}" pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/(\d+)/c(\d+))" - test = ("https://www.hbrowse.com/10363/c00000", { - "url": "6feefbc9f4b98e20d8425ddffa9dd111791dc3e6", - "keyword": "274996f6c809e5250b6ff3abbc5147e29f89d9a5", - "content": "44578ebbe176c2c27434966aef22945787e2781e", - }) + example = "https://www.hbrowse.com/12345/c00000" def __init__(self, match): self.path, self.gid, self.chapter = match.groups() @@ -75,10 +71,7 @@ class HbrowseMangaExtractor(HbrowseBase, MangaExtractor): chapterclass = HbrowseChapterExtractor reverse = False pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/\d+)/?$" - test = ("https://www.hbrowse.com/10363", { - "url": "b89682bfb86c11d2af0dc47463804ec3ac4aadd6", - "keyword": "4b15fda1858a69de1fbf5afddfe47dd893397312", - }) + example = "https://www.hbrowse.com/12345" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hentai2read.py b/gallery_dl/extractor/hentai2read.py index e771a4fc..9ab1411f 100644 --- a/gallery_dl/extractor/hentai2read.py +++ b/gallery_dl/extractor/hentai2read.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -23,31 +23,7 @@ class Hentai2readChapterExtractor(Hentai2readBase, ChapterExtractor): """Extractor for a single manga chapter from hentai2read.com""" archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+/([^/?#]+))" - test = ( - ("https://hentai2read.com/amazon_elixir/1/", { - "url": "964b942cf492b3a129d2fe2608abfc475bc99e71", - "keyword": "85645b02d34aa11b3deb6dadd7536863476e1bad", - }), - ("https://hentai2read.com/popuni_kei_joshi_panic/2.5/", { - "pattern": r"https://hentaicdn\.com/hentai" - r"/13088/2\.5y/ccdn00\d+\.jpg", - "count": 36, - "keyword": { - "author": "Kurisu", - "chapter": 2, - "chapter_id": 75152, - "chapter_minor": ".5", - "count": 36, - "lang": "en", - "language": "English", - "manga": "Popuni Kei Joshi Panic!", - "manga_id": 13088, - "page": int, - "title": "Popuni Kei Joshi Panic! 2.5", - "type": "Original", - }, - }), - ) + example = "https://hentai2read.com/TITLE/1/" def __init__(self, match): self.chapter = match.group(2) @@ -85,31 +61,7 @@ class Hentai2readMangaExtractor(Hentai2readBase, MangaExtractor): """Extractor for hmanga from hentai2read.com""" chapterclass = Hentai2readChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+)/?$" - test = ( - ("https://hentai2read.com/amazon_elixir/", { - "url": "273073752d418ec887d7f7211e42b832e8c403ba", - "keyword": "5c1b712258e78e120907121d3987c71f834d13e1", - }), - ("https://hentai2read.com/oshikage_riot/", { - "url": "6595f920a3088a15c2819c502862d45f8eb6bea6", - "keyword": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", - }), - ("https://hentai2read.com/popuni_kei_joshi_panic/", { - "pattern": Hentai2readChapterExtractor.pattern, - "range": "2-3", - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": ".5", - "lang": "en", - "language": "English", - "manga": "Popuni Kei Joshi Panic!", - "manga_id": 13088, - "title": str, - "type": "Original", - }, - }), - ) + example = "https://hentai2read.com/TITLE/" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hentaicosplays.py b/gallery_dl/extractor/hentaicosplays.py index ac03923f..62df1925 100644 --- a/gallery_dl/extractor/hentaicosplays.py +++ b/gallery_dl/extractor/hentaicosplays.py @@ -21,36 +21,7 @@ class HentaicosplaysGalleryExtractor(GalleryExtractor): pattern = r"((?:https?://)?(?:\w{2}\.)?" \ r"(hentai-cosplays|hentai-img|porn-images-xxx)\.com)/" \ r"(?:image|story)/([\w-]+)" - test = ( - ("https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", { - "pattern": r"https://static\d?.hentai-cosplays.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 18, - "site": "hentai-cosplays", - "slug": "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", - }, - }), - ("https://fr.porn-images-xxx.com/image/enako-enako-24/", { - "pattern": r"https://static\d?.porn-images-xxx.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 11, - "site": "porn-images-xxx", - "title": str, - }, - }), - ("https://ja.hentai-img.com/image/hollow-cora-502/", { - "pattern": r"https://static\d?.hentai-img.com/upload/" - r"\d+/\d+/\d+/\d+.jpg$", - "keyword": { - "count": 2, - "site": "hentai-img", - "title": str, - }, - }), - ) + example = "https://hentai-cosplays.com/image/TITLE/" def __init__(self, match): root, self.site, self.slug = match.groups() diff --git a/gallery_dl/extractor/hentaifoundry.py b/gallery_dl/extractor/hentaifoundry.py index 56ea1d4d..4c02000c 100644 --- a/gallery_dl/extractor/hentaifoundry.py +++ b/gallery_dl/extractor/hentaifoundry.py @@ -168,7 +168,7 @@ class HentaifoundryUserExtractor(HentaifoundryExtractor): """Extractor for a hentaifoundry user profile""" subcategory = "user" pattern = BASE_PATTERN + r"/user/([^/?#]+)/profile" - test = ("https://www.hentai-foundry.com/user/Tenpura/profile",) + example = "https://www.hentai-foundry.com/user/USER/profile" def initialize(self): pass @@ -192,12 +192,7 @@ class HentaifoundryPicturesExtractor(HentaifoundryExtractor): """Extractor for all pictures of a hentaifoundry user""" subcategory = "pictures" pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)(?:/page/(\d+))?/?$" - test = ( - ("https://www.hentai-foundry.com/pictures/user/Tenpura", { - "url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", - }), - ("https://www.hentai-foundry.com/pictures/user/Tenpura/page/3"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -209,13 +204,7 @@ class HentaifoundryScrapsExtractor(HentaifoundryExtractor): subcategory = "scraps" directory_fmt = ("{category}", "{user}", "Scraps") pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)/scraps" - test = ( - ("https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", { - "url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7", - }), - ("https://www.hentai-foundry.com" - "/pictures/user/Evulchibi/scraps/page/3"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER/scraps" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -229,13 +218,7 @@ class HentaifoundryFavoriteExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "{user}", "Favorites") archive_fmt = "f_{user}_{index}" pattern = BASE_PATTERN + r"/user/([^/?#]+)/faves/pictures" - test = ( - ("https://www.hentai-foundry.com/user/Tenpura/faves/pictures", { - "url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", - }), - ("https://www.hentai-foundry.com" - "/user/Tenpura/faves/pictures/page/3"), - ) + example = "https://www.hentai-foundry.com/user/USER/faves/pictures" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -249,10 +232,7 @@ class HentaifoundryRecentExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "Recent Pictures", "{date}") archive_fmt = "r_{index}" pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)" - test = ("https://www.hentai-foundry.com/pictures/recent/2018-09-20", { - "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "range": "20-30", - }) + example = "https://www.hentai-foundry.com/pictures/recent/1970-01-01" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -268,10 +248,7 @@ class HentaifoundryPopularExtractor(HentaifoundryExtractor): directory_fmt = ("{category}", "Popular Pictures") archive_fmt = "p_{index}" pattern = BASE_PATTERN + r"/pictures/popular()" - test = ("https://www.hentai-foundry.com/pictures/popular", { - "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "range": "20-30", - }) + example = "https://www.hentai-foundry.com/pictures/popular" def __init__(self, match): HentaifoundryExtractor.__init__(self, match) @@ -283,34 +260,8 @@ class HentaifoundryImageExtractor(HentaifoundryExtractor): subcategory = "image" pattern = (r"(https?://)?(?:www\.|pictures\.)?hentai-foundry\.com" r"/(?:pictures/user|[^/?#])/([^/?#]+)/(\d+)") - test = ( - (("https://www.hentai-foundry.com" - "/pictures/user/Tenpura/407501/shimakaze"), { - "url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3", - "content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", - "keyword": { - "artist" : "Tenpura", - "date" : "dt:2016-02-22 14:41:19", - "description": "Thank you!", - "height" : 700, - "index" : 407501, - "media" : "Other digital art", - "ratings": ["Sexual content", "Contains female nudity"], - "score" : int, - "tags" : ["collection", "kancolle", "kantai", "shimakaze"], - "title" : "shimakaze", - "user" : "Tenpura", - "views" : int, - "width" : 495, - }, - }), - ("http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", { - "pattern": "http://pictures.hentai-foundry.com/t/Tenpura/407501/", - }), - ("https://www.hentai-foundry.com/pictures/user/Tenpura/407501/"), - ("https://pictures.hentai-foundry.com" - "/t/Tenpura/407501/Tenpura-407501-shimakaze.png"), - ) + example = "https://www.hentai-foundry.com/pictures/user/USER/12345/TITLE" + skip = Extractor.skip def __init__(self, match): @@ -331,24 +282,7 @@ class HentaifoundryStoriesExtractor(HentaifoundryExtractor): subcategory = "stories" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)(?:/page/(\d+))?/?$" - test = ("https://www.hentai-foundry.com/stories/user/SnowWolf35", { - "count": ">= 35", - "keyword": { - "author" : "SnowWolf35", - "chapters" : int, - "comments" : int, - "date" : "type:datetime", - "description": str, - "index" : int, - "rating" : int, - "ratings" : list, - "status" : "re:(Inc|C)omplete", - "title" : str, - "user" : "SnowWolf35", - "views" : int, - "words" : int, - }, - }) + example = "https://www.hentai-foundry.com/stories/user/USER" def items(self): self._init_site_filters() @@ -367,11 +301,8 @@ class HentaifoundryStoryExtractor(HentaifoundryExtractor): subcategory = "story" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)" - test = (("https://www.hentai-foundry.com/stories/user/SnowWolf35" - "/26416/Overwatch-High-Chapter-Voting-Location"), { - "url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", - "keyword": {"title": "Overwatch High Chapter Voting Location"}, - }) + example = "https://www.hentai-foundry.com/stories/user/USER/12345/TITLE" + skip = Extractor.skip def __init__(self, match): diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py index a1e681d1..31a302d1 100644 --- a/gallery_dl/extractor/hentaifox.py +++ b/gallery_dl/extractor/hentaifox.py @@ -21,40 +21,7 @@ class HentaifoxBase(): class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): """Extractor for image galleries on hentaifox.com""" pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))" - test = ( - ("https://hentaifox.com/gallery/56622/", { - "pattern": r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", - "keyword": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", - "count": 24, - }), - # 'split_tag' element (#1378) - ("https://hentaifox.com/gallery/630/", { - "keyword": { - "artist": ["beti", "betty", "magi", "mimikaki"], - "characters": [ - "aerith gainsborough", - "tifa lockhart", - "yuffie kisaragi" - ], - "count": 32, - "gallery_id": 630, - "group": ["cu-little2"], - "parody": ["darkstalkers | vampire", "final fantasy vii"], - "tags": ["femdom", "fingering", "masturbation", "yuri"], - "title": "Cu-Little Bakanya～", - "type": "doujinshi", - }, - }), - # email-protected title (#4201) - ("https://hentaifox.com/gallery/35261/", { - "keyword": { - "gallery_id": 35261, - "title": "ManageM@ster!", - "artist": ["haritama hiroki"], - "group": ["studio n.ball"], - }, - }), - ) + example = "https://hentaifox.com/gallery/12345/" def __init__(self, match): GalleryExtractor.__init__(self, match) @@ -116,22 +83,7 @@ class HentaifoxSearchExtractor(HentaifoxBase, Extractor): subcategory = "search" pattern = (r"(?:https?://)?(?:www\.)?hentaifox\.com" r"(/(?:parody|tag|artist|character|search|group)/[^/?%#]+)") - test = ( - ("https://hentaifox.com/parody/touhou-project/"), - ("https://hentaifox.com/character/reimu-hakurei/"), - ("https://hentaifox.com/artist/distance/"), - ("https://hentaifox.com/search/touhou/"), - ("https://hentaifox.com/group/v-slash/"), - ("https://hentaifox.com/tag/heterochromia/", { - "pattern": HentaifoxGalleryExtractor.pattern, - "count": ">= 60", - "keyword": { - "url" : str, - "gallery_id": int, - "title" : str, - }, - }), - ) + example = "https://hentaifox.com/tag/TAG/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index 0617330b..f3f43c47 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -17,27 +17,7 @@ class HentaihandGalleryExtractor(GalleryExtractor): category = "hentaihand" root = "https://hentaihand.com" pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" - test = ( - (("https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-" - "no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-" - "no-railgun-english"), { - "pattern": r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", - "count": 50, - "keyword": { - "artists" : ["Takumi Na Muchi"], - "date" : "dt:2014-06-28 00:00:00", - "gallery_id": 37387, - "lang" : "en", - "language" : "English", - "parodies" : ["Toaru Kagaku No Railgun"], - "relationships": list, - "tags" : list, - "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", - "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", - "type" : "Doujinshi", - }, - }), - ) + example = "https://hentaihand.com/en/comic/TITLE" def __init__(self, match): self.slug = match.group(1) @@ -76,15 +56,7 @@ class HentaihandTagExtractor(Extractor): pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" r"/\w+/(parody|character|tag|artist|group|language" r"|category|relationship)/([^/?#]+)") - test = ( - ("https://hentaihand.com/en/artist/takumi-na-muchi", { - "pattern": HentaihandGalleryExtractor.pattern, - "count": ">= 6", - }), - ("https://hentaihand.com/en/tag/full-color"), - ("https://hentaihand.com/fr/language/japanese"), - ("https://hentaihand.com/zh/category/manga"), - ) + example = "https://hentaihand.com/en/tag/TAG" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py index 2297cc06..ba9558c0 100644 --- a/gallery_dl/extractor/hentaihere.py +++ b/gallery_dl/extractor/hentaihere.py @@ -23,32 +23,7 @@ class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor): """Extractor for a single manga chapter from hentaihere.com""" archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)" - test = ( - ("https://hentaihere.com/m/S13812/1/1/", { - "url": "964b942cf492b3a129d2fe2608abfc475bc99e71", - "keyword": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", - }), - ("https://hentaihere.com/m/S23048/1.5/1/", { - "pattern": r"https://hentaicdn\.com/hentai" - r"/23048/1\.5/ccdn00\d+\.jpg", - "count": 32, - "keyword": { - "author": "Shinozuka Yuuji", - "chapter": 1, - "chapter_id": 80186, - "chapter_minor": ".5", - "count": 32, - "lang": "en", - "language": "English", - "manga": "High School Slut's Love Consultation", - "manga_id": 23048, - "page": int, - "title": "High School Slut's Love Consultation + " - "Girlfriend [Full Color]", - "type": "Original", - }, - }), - ) + example = "https://hentaihere.com/m/S12345/1/1/" def __init__(self, match): self.manga_id, self.chapter = match.groups() @@ -87,26 +62,7 @@ class HentaihereMangaExtractor(HentaihereBase, MangaExtractor): """Extractor for hmanga from hentaihere.com""" chapterclass = HentaihereChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$" - test = ( - ("https://hentaihere.com/m/S13812", { - "url": "d1ba6e28bb2162e844f8559c2b2725ba0a093559", - "keyword": "5c1b712258e78e120907121d3987c71f834d13e1", - }), - ("https://hentaihere.com/m/S7608", { - "url": "6c5239758dc93f6b1b4175922836c10391b174f7", - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": "", - "lang": "en", - "language": "English", - "manga": "Oshikake Riot", - "manga_id": 7608, - "title": r"re:Oshikake Riot( \d+)?", - "type": "Original", - }, - }), - ) + example = "https://hentaihere.com/m/S12345" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index 22fe7f6c..32ca1519 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -31,7 +31,7 @@ class HiperdexBase(): return { "manga" : text.unescape(extr( - "", "<").rpartition(" - ")[0].strip()), + "<title>", "<").rpartition(" Manga - ")[0].strip()), "url" : text.unescape(extr( 'property="og:url" content="', '"')), "score" : text.parse_float(extr( @@ -69,30 +69,7 @@ class HiperdexBase(): class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor): """Extractor for manga chapters from hiperdex.com""" pattern = BASE_PATTERN + r"(/manga/([^/?#]+)/([^/?#]+))" - test = ( - ("https://hiperdex.com/manga/domestic-na-kanojo/154-5/", { - "pattern": r"https://(1st)?hiperdex\d?.(com|net|info)" - r"/wp-content/uploads/WP-manga/data" - r"/manga_\w+/[0-9a-f]{32}/\d+\.webp", - "count": 9, - "keyword": { - "artist" : "Sasuga Kei", - "author" : "Sasuga Kei", - "chapter": 154, - "chapter_minor": ".5", - "description": "re:Natsuo Fujii is in love with his teacher, ", - "genre" : list, - "manga" : "Domestic na Kanojo", - "release": 2014, - "score" : float, - "type" : "Manga", - }, - }), - ("https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex2.com/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex.net/manga/domestic-na-kanojo/154-5/"), - ("https://hiperdex.info/manga/domestic-na-kanojo/154-5/"), - ) + example = "https://hiperdex.com/manga/MANGA/CHAPTER/" def __init__(self, match): root, path, self.manga, self.chapter = match.groups() @@ -114,30 +91,7 @@ class HiperdexMangaExtractor(HiperdexBase, MangaExtractor): """Extractor for manga from hiperdex.com""" chapterclass = HiperdexChapterExtractor pattern = BASE_PATTERN + r"(/manga/([^/?#]+))/?$" - test = ( - ("https://hiperdex.com/manga/1603231576-youre-not-that-special/", { - "count": 51, - "pattern": HiperdexChapterExtractor.pattern, - "keyword": { - "artist" : "Bolp", - "author" : "Abyo4", - "chapter": int, - "chapter_minor": "", - "description": "re:I didn’t think much of the creepy girl in ", - "genre" : list, - "manga" : "You’re Not That Special!", - "release": 2019, - "score" : float, - "status" : "Completed", - "type" : "Manhwa", - }, - }), - ("https://hiperdex.com/manga/youre-not-that-special/"), - ("https://1sthiperdex.com/manga/youre-not-that-special/"), - ("https://hiperdex2.com/manga/youre-not-that-special/"), - ("https://hiperdex.net/manga/youre-not-that-special/"), - ("https://hiperdex.info/manga/youre-not-that-special/"), - ) + example = "https://hiperdex.com/manga/MANGA/" def __init__(self, match): root, path, self.manga = match.groups() @@ -173,16 +127,7 @@ class HiperdexArtistExtractor(HiperdexBase, MangaExtractor): chapterclass = HiperdexMangaExtractor reverse = False pattern = BASE_PATTERN + r"(/manga-a(?:rtist|uthor)/(?:[^/?#]+))" - test = ( - ("https://1sthiperdex.com/manga-artist/beck-ho-an/"), - ("https://hiperdex.net/manga-artist/beck-ho-an/"), - ("https://hiperdex2.com/manga-artist/beck-ho-an/"), - ("https://hiperdex.info/manga-artist/beck-ho-an/"), - ("https://hiperdex.com/manga-author/viagra/", { - "pattern": HiperdexMangaExtractor.pattern, - "count": ">= 6", - }), - ) + example = "https://hiperdex.com/manga-artist/NAME/" def __init__(self, match): self.root = text.ensure_http_scheme(match.group(1)) diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index c012c665..bc49ca31 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -23,47 +23,7 @@ class HitomiGalleryExtractor(GalleryExtractor): pattern = (r"(?:https?://)?hitomi\.la" r"/(?:manga|doujinshi|cg|gamecg|galleries|reader)" r"/(?:[^/?#]+-)?(\d+)") - test = ( - ("https://hitomi.la/galleries/867789.html", { - "pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+" - r"/[0-9a-f]{64}\.webp", - "keyword": "86af5371f38117a07407f11af689bdd460b09710", - "count": 16, - }), - # download test - ("https://hitomi.la/galleries/1401410.html", { - "range": "1", - "content": "d75d5a3d1302a48469016b20e53c26b714d17745", - }), - # Game CG with scenes (#321) - ("https://hitomi.la/galleries/733697.html", { - "count": 210, - }), - # fallback for galleries only available through /reader/ URLs - ("https://hitomi.la/galleries/1045954.html", { - "count": 1413, - }), - # gallery with "broken" redirect - ("https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", { - "count": 10, - "options": (("format", "original"),), - "pattern": r"https://[a-c]b\.hitomi\.la/images/\d+/\d+" - r"/[0-9a-f]{64}\.jpg", - }), - # no tags - ("https://hitomi.la/cg/1615823.html", { - "count": 22, - "options": (("format", "avif"),), - "pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+" - r"/[0-9a-f]{64}\.avif", - }), - ("https://hitomi.la/manga/amazon-no-hiyaku-867789.html"), - ("https://hitomi.la/manga/867789.html"), - ("https://hitomi.la/doujinshi/867789.html"), - ("https://hitomi.la/cg/867789.html"), - ("https://hitomi.la/gamecg/867789.html"), - ("https://hitomi.la/reader/867789.html"), - ) + example = "https://hitomi.la/manga/TITLE-867789.html" def __init__(self, match): self.gid = match.group(1) @@ -149,17 +109,7 @@ class HitomiTagExtractor(Extractor): pattern = (r"(?:https?://)?hitomi\.la/" r"(tag|artist|group|series|type|character)/" r"([^/?#]+)\.html") - test = ( - ("https://hitomi.la/tag/screenshots-japanese.html", { - "pattern": HitomiGalleryExtractor.pattern, - "count": ">= 35", - }), - ("https://hitomi.la/artist/a1-all-1.html"), - ("https://hitomi.la/group/initial%2Dg-all-1.html"), - ("https://hitomi.la/series/amnesia-all-1.html"), - ("https://hitomi.la/type/doujinshi-all-1.html"), - ("https://hitomi.la/character/a2-all-1.html"), - ) + example = "https://hitomi.la/tag/TAG-LANG.html" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/hotleak.py b/gallery_dl/extractor/hotleak.py index 2ab9f3cc..6d3184d9 100644 --- a/gallery_dl/extractor/hotleak.py +++ b/gallery_dl/extractor/hotleak.py @@ -21,9 +21,6 @@ class HotleakExtractor(Extractor): archive_fmt = "{type}_{creator}_{id}" root = "https://hotleak.vip" - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): for post in self.posts(): yield Message.Directory, post @@ -59,30 +56,7 @@ class HotleakPostExtractor(HotleakExtractor): subcategory = "post" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/]+)/(photo|video)/(\d+)") - test = ( - ("https://hotleak.vip/kaiyakawaii/photo/1617145", { - "pattern": r"https://hotleak\.vip/storage/images/3625" - r"/1617145/fefdd5988dfcf6b98cc9e11616018868\.jpg", - "keyword": { - "id": 1617145, - "creator": "kaiyakawaii", - "type": "photo", - "filename": "fefdd5988dfcf6b98cc9e11616018868", - "extension": "jpg", - }, - }), - ("https://hotleak.vip/lilmochidoll/video/1625538", { - "pattern": r"ytdl:https://cdn8-leak\.camhdxx\.com" - r"/1661/1625538/index\.m3u8", - "keyword": { - "id": 1625538, - "creator": "lilmochidoll", - "type": "video", - "filename": "index", - "extension": "mp4", - }, - }), - ) + example = "https://hotleak.vip/MODEL/photo/12345" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -118,18 +92,7 @@ class HotleakCreatorExtractor(HotleakExtractor): subcategory = "creator" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/?#]+)/?$") - test = ( - ("https://hotleak.vip/kaiyakawaii", { - "range": "1-200", - "count": 200, - }), - ("https://hotleak.vip/stellaviolet", { - "count": "> 600" - }), - ("https://hotleak.vip/doesnotexist", { - "exception": exception.NotFoundError, - }), - ) + example = "https://hotleak.vip/MODEL" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -182,20 +145,7 @@ class HotleakCategoryExtractor(HotleakExtractor): """Extractor for hotleak categories""" subcategory = "category" pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?" - test = ( - ("https://hotleak.vip/photos", { - "pattern": HotleakPostExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://hotleak.vip/videos"), - ("https://hotleak.vip/creators", { - "pattern": HotleakCreatorExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://hotleak.vip/hot"), - ) + example = "https://hotleak.vip/photos" def __init__(self, match): HotleakExtractor.__init__(self, match) @@ -217,14 +167,7 @@ class HotleakSearchExtractor(HotleakExtractor): """Extractor for hotleak search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))" - test = ( - ("https://hotleak.vip/search?search=gallery-dl", { - "count": 0, - }), - ("https://hotleak.vip/search?search=hannah", { - "count": "> 30", - }), - ) + example = "https://hotleak.vip/search?search=QUERY" def __init__(self, match): HotleakExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py index fcac7fe2..16e40970 100644 --- a/gallery_dl/extractor/idolcomplex.py +++ b/gallery_dl/extractor/idolcomplex.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2021 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -133,20 +133,7 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/\?([^#]*)" - test = ( - ("https://idol.sankakucomplex.com/?tags=lyumos", { - "count": 5, - "range": "18-22", - "pattern": r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" - r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", - }), - ("https://idol.sankakucomplex.com/?tags=order:favcount", { - "count": 5, - "range": "18-22", - }), - ("https://idol.sankakucomplex.com" - "/?tags=lyumos+wreath&page=3&next=694215"), - ) + example = "https://idol.sankakucomplex.com/?tags=TAGS" per_page = 20 def __init__(self, match): @@ -214,9 +201,7 @@ class IdolcomplexPoolExtractor(IdolcomplexExtractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)" - test = ("https://idol.sankakucomplex.com/pool/show/145", { - "count": 3, - }) + example = "https://idol.sankakucomplex.com/pool/show/12345" per_page = 24 def __init__(self, match): @@ -251,17 +236,7 @@ class IdolcomplexPostExtractor(IdolcomplexExtractor): subcategory = "post" archive_fmt = "{id}" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)" - test = ("https://idol.sankakucomplex.com/post/show/694215", { - "content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - "options": (("tags", True),), - "keyword": { - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_idol": str, - "tags_medium": str, - "tags_general": str, - }, - }) + example = "https://idol.sankakucomplex.com/post/show/12345" def __init__(self, match): IdolcomplexExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imagebam.py b/gallery_dl/extractor/imagebam.py index 9a3ea368..68360e96 100644 --- a/gallery_dl/extractor/imagebam.py +++ b/gallery_dl/extractor/imagebam.py @@ -9,7 +9,7 @@ """Extractors for https://www.imagebam.com/""" from .common import Extractor, Message -from .. import text, exception +from .. import text import re @@ -46,26 +46,7 @@ class ImagebamGalleryExtractor(ImagebamExtractor): archive_fmt = "{gallery_key}_{image_key}" pattern = (r"(?:https?://)?(?:www\.)?imagebam\.com" r"(/(?:gallery/|view/G)[a-zA-Z0-9]+)") - test = ( - ("https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", { - "url": "76d976788ae2757ac81694736b07b72356f5c4c8", - "keyword": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", - "content": "596e6bfa157f2c7169805d50075c2986549973a8", - }), - ("http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", { - # more than 100 images; see issue #219 - "count": 107, - "url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", - }), - ("http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", { - "exception": exception.HttpError, - }), - # /view/ path (#2378) - ("https://www.imagebam.com/view/GA3MT1", { - "url": "35018ce1e00a2d2825a33d3cd37857edaf804919", - "keyword": "3a9f98178f73694c527890c0d7ca9a92b46987ba", - }), - ) + example = "https://www.imagebam.com/view/GID" def items(self): page = self.request(self.root + self.path).text @@ -110,24 +91,7 @@ class ImagebamImageExtractor(ImagebamExtractor): archive_fmt = "{image_key}" pattern = (r"(?:https?://)?(?:\w+\.)?imagebam\.com" r"(/(?:image/|view/M|(?:[0-9a-f]{2}/){3})[a-zA-Z0-9]+)") - test = ( - ("https://www.imagebam.com/image/94d56c502511890", { - "url": "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", - "keyword": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("http://images3.imagebam.com/1d/8c/44/94d56c502511890.png"), - # NSFW (#1534) - ("https://www.imagebam.com/image/0850951366904951", { - "url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", - }), - # /view/ path (#2378) - ("https://www.imagebam.com/view/ME8JOQP", { - "url": "4dca72bbe61a0360185cf4ab2bed8265b49565b8", - "keyword": "15a494c02fd30846b41b42a26117aedde30e4ceb", - "content": "f81008666b17a42d8834c4749b910e1dc10a6e83", - }), - ) + example = "https://www.imagebam.com/view/MID" def items(self): path = self.path diff --git a/gallery_dl/extractor/imagechest.py b/gallery_dl/extractor/imagechest.py index 2babead8..9aa03329 100644 --- a/gallery_dl/extractor/imagechest.py +++ b/gallery_dl/extractor/imagechest.py @@ -18,29 +18,7 @@ class ImagechestGalleryExtractor(GalleryExtractor): category = "imagechest" root = "https://imgchest.com" pattern = r"(?:https?://)?(?:www\.)?imgchest\.com/p/([A-Za-z0-9]{11})" - test = ( - ("https://imgchest.com/p/3na7kr3by8d", { - "pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "keyword": { - "count": 3, - "gallery_id": "3na7kr3by8d", - "num": int, - "title": "Wizardry - Video Game From The Mid 80's", - }, - "url": "7328ca4ec2459378d725e3be19f661d2b045feda", - "content": "076959e65be30249a2c651fbe6090dc30ba85193", - "count": 3 - }), - # "Load More Files" button (#4028) - ("https://imgchest.com/p/9p4n3q2z7nq", { - "pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", - "count": 52, - }), - ("https://imgchest.com/p/xxxxxxxxxxx", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgchest.com/p/abcdefghijk" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py index 43ac3a35..aca101e1 100644 --- a/gallery_dl/extractor/imagefap.py +++ b/gallery_dl/extractor/imagefap.py @@ -23,9 +23,6 @@ class ImagefapExtractor(Extractor): archive_fmt = "{gallery_id}_{image_id}" request_interval = (2.0, 4.0) - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) @@ -43,50 +40,7 @@ class ImagefapGalleryExtractor(ImagefapExtractor): """Extractor for image galleries from imagefap.com""" subcategory = "gallery" pattern = BASE_PATTERN + r"/(?:gallery\.php\?gid=|gallery/|pictures/)(\d+)" - - test = ( - ("https://www.imagefap.com/gallery/7102714", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/\d+/\d+/\d+\.jpg", - "keyword": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b", - "content": "694a0a57385980a6f90fbc296cadcd6c11ba2dab", - }), - ("https://www.imagefap.com/gallery/7876223", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/\d+/\d+/\d+\.jpg", - "keyword": { - "categories": ["Asses", "Softcore", "Pornstars"], - "count": 44, - "description": "", - "gallery_id": 7876223, - "image_id": int, - "num": int, - "tags": ["big ass", "panties", "horny", - "pussy", "exposed", "outdoor"], - "title": "Kelsi Monroe in lingerie", - "uploader": "BdRachel", - }, - "count": 44, - }), - # description (#3905) - ("https://www.imagefap.com/gallery/6180555", { - "range": "1", - "keyword": { - "categories": ["Amateur", "Softcore", "Homemade"], - "count": 36, - "description": "Nude and dressed sluts showing off the goods", - "gallery_id": 6180555, - "image_id": int, - "num": int, - "tags": [] , - "title": "Dressed or Undressed MG*", - "uploader": "splitopen", - }, - }), - ("https://www.imagefap.com/pictures/7102714"), - ("https://www.imagefap.com/gallery.php?gid=7102714"), - ("https://beta.imagefap.com/gallery.php?gid=7102714"), - ) + example = "https://www.imagefap.com/gallery/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -157,22 +111,7 @@ class ImagefapImageExtractor(ImagefapExtractor): """Extractor for single images from imagefap.com""" subcategory = "image" pattern = BASE_PATTERN + r"/photo/(\d+)" - test = ( - ("https://www.imagefap.com/photo/1962981893", { - "pattern": r"https://cdnh?\.imagefap\.com" - r"/images/full/65/196/1962981893\.jpg", - "keyword": { - "date": "21/08/2014", - "gallery_id": 7876223, - "height": 1600, - "image_id": 1962981893, - "title": "Kelsi Monroe in lingerie", - "uploader": "BdRachel", - "width": 1066, - }, - }), - ("https://beta.imagefap.com/photo/1962981893"), - ) + example = "https://www.imagefap.com/photo/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -213,35 +152,7 @@ class ImagefapFolderExtractor(ImagefapExtractor): pattern = (BASE_PATTERN + r"/(?:organizer/|" r"(?:usergallery\.php\?user(id)?=([^&#]+)&" r"|profile/([^/?#]+)/galleries\?)folderid=)(\d+|-1)") - test = ( - ("https://www.imagefap.com/organizer/409758", { - "pattern": r"https://www\.imagefap\.com/gallery/7876223", - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - "count": 1, - }), - (("https://www.imagefap.com/usergallery.php" - "?userid=1981976&folderid=409758"), { - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - }), - (("https://www.imagefap.com/usergallery.php" - "?user=BdRachel&folderid=409758"), { - "url": "37822523e6e4a56feb9dea35653760c86b44ff89", - }), - ("https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - (("https://www.imagefap.com/usergallery.php" - "?userid=1981976&folderid=-1"), { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - (("https://www.imagefap.com/usergallery.php" - "?user=BdRachel&folderid=-1"), { - "pattern": ImagefapGalleryExtractor.pattern, - "range": "1-40", - }), - ) + example = "https://www.imagefap.com/organizer/12345" def __init__(self, match): ImagefapExtractor.__init__(self, match) @@ -293,20 +204,7 @@ class ImagefapUserExtractor(ImagefapExtractor): pattern = (BASE_PATTERN + r"/(?:profile(?:\.php\?user=|/)([^/?#]+)(?:/galleries)?" r"|usergallery\.php\?userid=(\d+))(?:$|#)") - test = ( - ("https://www.imagefap.com/profile/BdRachel", { - "pattern": ImagefapFolderExtractor.pattern, - "count": ">= 18", - }), - ("https://www.imagefap.com/usergallery.php?userid=1862791", { - "pattern": r"https://www\.imagefap\.com" - r"/profile/LucyRae/galleries\?folderid=-1", - "count": 1, - }), - ("https://www.imagefap.com/profile/BdRachel/galleries"), - ("https://www.imagefap.com/profile.php?user=BdRachel"), - ("https://beta.imagefap.com/profile.php?user=BdRachel"), - ) + example = "https://www.imagefap.com/profile/USER" def __init__(self, match): ImagefapExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 8ef51b0a..94019bd6 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -74,34 +74,7 @@ class ImxtoImageExtractor(ImagehostImageExtractor): category = "imxto" pattern = (r"(?:https?://)?(?:www\.)?((?:imx\.to|img\.yt)" r"/(?:i/|img-)(\w+)(\.html)?)") - test = ( - ("https://imx.to/i/1qdeva", { # new-style URL - "url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "size" : 18, - "width" : 64, - "height": 32, - "hash" : "94d56c599223c59f3feb71ea603484d1", - }, - }), - ("https://imx.to/img-57a2050547b97.html", { # old-style URL - "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - "content": "54592f2635674c25677c6872db3709d343cdf92f", - "keyword": { - "size" : 5284, - "width" : 320, - "height": 160, - "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", - }, - }), - ("https://img.yt/img-57a2050547b97.html", { # img.yt domain - "url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - }), - ("https://imx.to/img-57a2050547b98.html", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imx.to/i/ID" _params = "simple" _encoding = "utf-8" @@ -140,11 +113,7 @@ class ImxtoGalleryExtractor(ImagehostImageExtractor): category = "imxto" subcategory = "gallery" pattern = r"(?:https?://)?(?:www\.)?(imx\.to/g/([^/?#]+))" - test = ("https://imx.to/g/ozdy", { - "pattern": ImxtoImageExtractor.pattern, - "keyword": {"title": "untitled gallery"}, - "count": 40, - }) + example = "https://imx.to/g/ID" def items(self): page = self.request(self.page_url).text @@ -162,11 +131,7 @@ class AcidimgImageExtractor(ImagehostImageExtractor): """Extractor for single images from acidimg.cc""" category = "acidimg" pattern = r"(?:https?://)?((?:www\.)?acidimg\.cc/img-([a-z0-9]+)\.html)" - test = ("https://acidimg.cc/img-5acb6b9de4640.html", { - "url": "f132a630006e8d84f52d59555191ed82b3b64c04", - "keyword": "135347ab4345002fc013863c0d9419ba32d98f78", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://acidimg.cc/img-abc123.html" _params = "simple" _encoding = "utf-8" @@ -189,26 +154,13 @@ class ImagevenueImageExtractor(ImagehostImageExtractor): category = "imagevenue" pattern = (r"(?:https?://)?((?:www|img\d+)\.imagevenue\.com" r"/([A-Z0-9]{8,10}|view/.*|img\.php\?.*))") - test = ( - ("https://www.imagevenue.com/ME13LS07", { - "pattern": r"https://cdn-images\.imagevenue\.com" - r"/10/ac/05/ME13LS07_o\.png", - "keyword": "ae15d6e3b2095f019eee84cd896700cd34b09c36", - "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", - }), - (("https://www.imagevenue.com/view/o?i=92518_13732377" - "annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486"), { - "url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", - }), - (("http://img28116.imagevenue.com/img.php" - "?image=th_52709_test_122_64lo.jpg"), { - "url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", - }), - ) + example = "https://www.imagevenue.com/ME123456789" def get_info(self, page): pos = page.index('class="card-body') url, pos = text.extract(page, '<img src="', '"', pos) + if url.endswith("/loader.svg"): + url, pos = text.extract(page, '<img src="', '"', pos) filename, pos = text.extract(page, 'alt="', '"', pos) return url, text.unescape(filename) @@ -218,17 +170,7 @@ class ImagetwistImageExtractor(ImagehostImageExtractor): category = "imagetwist" pattern = (r"(?:https?://)?((?:www\.|phun\.)?" r"image(?:twist|haha)\.com/([a-z0-9]{12}))") - test = ( - ("https://imagetwist.com/f1i2s4vhvbrq/test.png", { - "url": "8d5e168c0bee30211f821c6f3b2116e419d42671", - "keyword": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("https://www.imagetwist.com/f1i2s4vhvbrq/test.png"), - ("https://phun.imagetwist.com/f1i2s4vhvbrq/test.png"), - ("https://imagehaha.com/f1i2s4vhvbrq/test.png"), - ("https://www.imagehaha.com/f1i2s4vhvbrq/test.png"), - ) + example = "https://imagetwist.com/123456abcdef/NAME.EXT" @property @memcache(maxage=3*3600) @@ -245,11 +187,7 @@ class ImgspiceImageExtractor(ImagehostImageExtractor): """Extractor for single images from imgspice.com""" category = "imgspice" pattern = r"(?:https?://)?((?:www\.)?imgspice\.com/([^/?#]+))" - test = ("https://imgspice.com/nwfwtpyog50y/test.png.html", { - "url": "b8c30a8f51ee1012959a4cfd46197fabf14de984", - "keyword": "100e310a19a2fa22d87e1bbc427ecb9f6501e0c0", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://imgspice.com/ID/NAME.EXT.html" def get_info(self, page): pos = page.find('id="imgpreview"') @@ -265,11 +203,7 @@ class PixhostImageExtractor(ImagehostImageExtractor): category = "pixhost" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" r"/show/\d+/(\d+)_[^/?#]+)") - test = ("http://pixhost.to/show/190/130327671_test-.png", { - "url": "4e5470dcf6513944773044d40d883221bbc46cff", - "keyword": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://pixhost.to/show/123/12345_NAME.EXT" _cookies = {"pixhostads": "1", "pixhosttest": "1"} def get_info(self, page): @@ -284,10 +218,7 @@ class PixhostGalleryExtractor(ImagehostImageExtractor): subcategory = "gallery" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" r"/gallery/([^/?#]+))") - test = ("https://pixhost.to/gallery/jSMFq", { - "pattern": PixhostImageExtractor.pattern, - "count": 3, - }) + example = "https://pixhost.to/gallery/ID" def items(self): page = text.extr(self.request( @@ -300,13 +231,9 @@ class PixhostGalleryExtractor(ImagehostImageExtractor): class PostimgImageExtractor(ImagehostImageExtractor): """Extractor for single images from postimages.org""" category = "postimg" - pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" - r"/(?!gallery/)(?:image/)?([^/?#]+)/?)") - test = ("https://postimg.cc/Wtn2b3hC", { - "url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", - "keyword": "2d05808d04e4e83e33200db83521af06e3147a84", - "content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", - }) + pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" + r"\.(?:cc|org)/(?!gallery/)(?:image/)?([^/?#]+)/?)") + example = "https://postimages.org/ID" def get_info(self, page): pos = page.index(' id="download"') @@ -319,12 +246,9 @@ class PostimgGalleryExtractor(ImagehostImageExtractor): """Extractor for images galleries from postimages.org""" category = "postimg" subcategory = "gallery" - pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" - r"/(?:gallery/)([^/?#]+)/?)") - test = ("https://postimg.cc/gallery/wxpDLgX", { - "pattern": PostimgImageExtractor.pattern, - "count": 22, - }) + pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" + r"\.(?:cc|org)/gallery/([^/?#]+))") + example = "https://postimages.org/gallery/ID" def items(self): page = self.request(self.page_url).text @@ -338,11 +262,7 @@ class TurboimagehostImageExtractor(ImagehostImageExtractor): category = "turboimagehost" pattern = (r"(?:https?://)?((?:www\.)?turboimagehost\.com" r"/p/(\d+)/[^/?#]+\.html)") - test = ("https://www.turboimagehost.com/p/39078423/test--.png.html", { - "url": "b94de43612318771ced924cb5085976f13b3b90e", - "keyword": "704757ca8825f51cec516ec44c1e627c1f2058ca", - "content": "f38b54b17cd7462e687b58d83f00fca88b1b105a", - }) + example = "https://www.turboimagehost.com/p/12345/NAME.EXT.html" def get_info(self, page): url = text.extract(page, 'src="', '"', page.index("<img "))[0] @@ -353,10 +273,7 @@ class ViprImageExtractor(ImagehostImageExtractor): """Extractor for single images from vipr.im""" category = "vipr" pattern = r"(?:https?://)?(vipr\.im/(\w+))" - test = ("https://vipr.im/kcd5jcuhgs3v.html", { - "url": "88f6a3ecbf3356a11ae0868b518c60800e070202", - "keyword": "c432e8a1836b0d97045195b745731c2b1bb0e771", - }) + example = "https://vipr.im/abc123.html" def get_info(self, page): url = text.extr(page, '<img src="', '"') @@ -367,11 +284,7 @@ class ImgclickImageExtractor(ImagehostImageExtractor): """Extractor for single images from imgclick.net""" category = "imgclick" pattern = r"(?:https?://)?((?:www\.)?imgclick\.net/([^/?#]+))" - test = ("http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", { - "url": "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", - "keyword": "6895256143eab955622fc149aa367777a8815ba3", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "http://imgclick.net/abc123/NAME.EXT.html" _https = False _params = "complex" @@ -385,11 +298,7 @@ class FappicImageExtractor(ImagehostImageExtractor): """Extractor for single images from fappic.com""" category = "fappic" pattern = r"(?:https?://)?((?:www\.)?fappic\.com/(\w+)/[^/?#]+)" - test = ("https://www.fappic.com/98wxqcklyh8k/test.png", { - "pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", - "keyword": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }) + example = "https://fappic.com/abc123/NAME.EXT" def get_info(self, page): url , pos = text.extract(page, '<a href="#"><img src="', '"') diff --git a/gallery_dl/extractor/imgbb.py b/gallery_dl/extractor/imgbb.py index ee979a65..1b741802 100644 --- a/gallery_dl/extractor/imgbb.py +++ b/gallery_dl/extractor/imgbb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -114,27 +114,7 @@ class ImgbbAlbumExtractor(ImgbbExtractor): subcategory = "album" directory_fmt = ("{category}", "{user}", "{album_name} {album_id}") pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?" - test = ( - ("https://ibb.co/album/i5PggF", { - "range": "1-80", - "url": "70afec9fcc3a6de62a6b644b487d892d8d47cf1a", - "keyword": "569e1d88ebdd27655387559cdf1cd526a3e1ab69", - }), - ("https://ibb.co/album/i5PggF?sort=title_asc", { - "range": "1-80", - "url": "afdf5fc95d8e09d77e8f44312f3e9b843987bb5a", - "keyword": "f090e14d0e5f7868595082b2c95da1309c84872d", - }), - # no user data (#471) - ("https://ibb.co/album/kYKpwF", { - "url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", - "keyword": {"user": ""}, - }), - # private - ("https://ibb.co/album/hqgWrF", { - "exception": exception.HttpError, - }), - ) + example = "https://ibb.co/album/ID" def __init__(self, match): ImgbbExtractor.__init__(self, match) @@ -169,10 +149,7 @@ class ImgbbUserExtractor(ImgbbExtractor): """Extractor for user profiles in imgbb.com""" subcategory = "user" pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?$" - test = ("https://folkie.imgbb.com", { - "range": "1-80", - "pattern": r"https?://i\.ibb\.co/\w+/[^/?#]+", - }) + example = "https://USER.imgbb.com" def __init__(self, match): ImgbbExtractor.__init__(self, match) @@ -196,19 +173,7 @@ class ImgbbUserExtractor(ImgbbExtractor): class ImgbbImageExtractor(ImgbbExtractor): subcategory = "image" pattern = r"(?:https?://)?ibb\.co/(?!album/)([^/?#]+)" - test = ("https://ibb.co/fUqh5b", { - "pattern": r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", - "content": "c5a0965178a8b357acd8aa39660092918c63795e", - "keyword": { - "id" : "fUqh5b", - "title" : "Arundel Ireeman 5", - "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", - "width" : 960, - "height": 719, - "user" : "folkie", - "extension": "jpg", - }, - }) + example = "https://ibb.co/ID" def __init__(self, match): ImgbbExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imgbox.py b/gallery_dl/extractor/imgbox.py index 530c4e1c..70697179 100644 --- a/gallery_dl/extractor/imgbox.py +++ b/gallery_dl/extractor/imgbox.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2019 Mike Fährmann +# Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from galleries at https://imgbox.com/""" +"""Extractors for https://imgbox.com/""" from .common import Extractor, Message, AsynchronousMixin from .. import text, exception @@ -63,20 +63,7 @@ class ImgboxGalleryExtractor(AsynchronousMixin, ImgboxExtractor): filename_fmt = "{num:>03}-{filename}.{extension}" archive_fmt = "{gallery_key}_{image_key}" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/g/([A-Za-z0-9]{10})" - test = ( - ("https://imgbox.com/g/JaX5V5HX7g", { - "url": "da4f15b161461119ee78841d4b8e8d054d95f906", - "keyword": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", - "content": "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", - }), - ("https://imgbox.com/g/cUGEkRbdZZ", { - "url": "76506a3aab175c456910851f66227e90484ca9f7", - "keyword": "fb0427b87983197849fb2887905e758f3e50cb6e", - }), - ("https://imgbox.com/g/JaX5V5HX7h", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgbox.com/g/12345abcde" def __init__(self, match): ImgboxExtractor.__init__(self, match) @@ -106,16 +93,7 @@ class ImgboxImageExtractor(ImgboxExtractor): subcategory = "image" archive_fmt = "{image_key}" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/([A-Za-z0-9]{8})" - test = ( - ("https://imgbox.com/qHhw7lpG", { - "url": "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", - "keyword": "dfc72310026b45f3feb4f9cada20c79b2575e1af", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - }), - ("https://imgbox.com/qHhw7lpH", { - "exception": exception.NotFoundError, - }), - ) + example = "https://imgbox.com/1234abcd" def __init__(self, match): ImgboxExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/imgth.py b/gallery_dl/extractor/imgth.py index 9ae22a91..3aa79229 100644 --- a/gallery_dl/extractor/imgth.py +++ b/gallery_dl/extractor/imgth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -17,24 +17,7 @@ class ImgthGalleryExtractor(GalleryExtractor): category = "imgth" root = "https://imgth.com" pattern = r"(?:https?://)?(?:www\.)?imgth\.com/gallery/(\d+)" - test = ( - ("https://imgth.com/gallery/37/wallpaper-anime", { - "url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", - "pattern": r"https://imgth\.com/images/2009/11/25" - r"/wallpaper-anime_\w+\.jpg", - "keyword": { - "count": 12, - "date": "dt:2009-11-25 18:21:00", - "extension": "jpg", - "filename": r"re:wallpaper-anime_\w+", - "gallery_id": 37, - "num": int, - "title": "Wallpaper anime", - "user": "celebrities", - }, - }), - ("https://www.imgth.com/gallery/37/wallpaper-anime"), - ) + example = "https://imgth.com/gallery/123/TITLE" def __init__(self, match): self.gallery_id = gid = match.group(1) diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index ca9671c3..8884d3ee 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -11,7 +11,6 @@ from .common import Extractor, Message from .. import text, exception - BASE_PATTERN = r"(?:https?://)?(?:www\.|[im]\.)?imgur\.(?:com|io)" @@ -65,69 +64,7 @@ class ImgurImageExtractor(ImgurExtractor): archive_fmt = "{id}" pattern = (BASE_PATTERN + r"/(?!gallery|search)" r"(?:r/\w+/)?(\w{7}|\w{5})[sbtmlh]?") - test = ( - ("https://imgur.com/21yMxCS", { - "url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", - "content": "0c8768055e4e20e7c7259608b67799171b691140", - "keyword": { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "21yMxCS", - "date" : "dt:2016-11-10 14:24:35", - "description" : "", - "downvote_count": int, - "duration" : 0, - "ext" : "png", - "favorite" : False, - "favorite_count": 0, - "has_sound" : False, - "height" : 32, - "id" : "21yMxCS", - "image_count" : 1, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : False, - "is_animated" : False, - "is_looping" : False, - "is_mature" : False, - "is_pending" : False, - "mime_type" : "image/png", - "name" : "test-テスト", - "point_count" : int, - "privacy" : "", - "score" : int, - "size" : 182, - "title" : "Test", - "upvote_count" : int, - "url" : "https://i.imgur.com/21yMxCS.png", - "view_count" : int, - "width" : 64, - }, - }), - ("http://imgur.com/0gybAXR", { # gifv/mp4 video - "url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", - "content": "a3c080e43f58f55243ab830569ba02309d59abfc", - }), - ("https://imgur.com/XFfsmuC", { # missing title in API response (#467) - "keyword": {"title": "Tears are a natural response to irritants"}, - }), - ("https://imgur.com/1Nily2P", { # animated png - "pattern": "https://i.imgur.com/1Nily2P.png", - }), - ("https://imgur.com/zzzzzzz", { # not found - "exception": exception.HttpError, - }), - ("https://m.imgur.com/r/Celebs/iHJ7tsM"), - ("https://www.imgur.com/21yMxCS"), # www - ("https://m.imgur.com/21yMxCS"), # mobile - ("https://imgur.com/zxaY6"), # 5 character key - ("https://imgur.io/zxaY6"), # .io - ("https://i.imgur.com/21yMxCS.png"), # direct link - ("https://i.imgur.io/21yMxCS.png"), # direct link .io - ("https://i.imgur.com/21yMxCSh.png"), # direct link thumbnail - ("https://i.imgur.com/zxaY6.gif"), # direct link (short) - ("https://i.imgur.com/zxaY6s.gif"), # direct link (short; thumb) - ) + example = "https://imgur.com/abcdefg" def items(self): image = self.api.image(self.key) @@ -152,71 +89,7 @@ class ImgurAlbumExtractor(ImgurExtractor): filename_fmt = "{category}_{album[id]}_{num:>03}_{id}.{extension}" archive_fmt = "{album[id]}_{id}" pattern = BASE_PATTERN + r"/a/(\w{7}|\w{5})" - test = ( - ("https://imgur.com/a/TcBmP", { - "url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", - "keyword": { - "album": { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "693j2Kr", - "date" : "dt:2015-10-09 10:37:50", - "description" : "", - "downvote_count": 0, - "favorite" : False, - "favorite_count": 0, - "id" : "TcBmP", - "image_count" : 19, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : True, - "is_mature" : False, - "is_pending" : False, - "privacy" : "private", - "score" : int, - "title" : "138", - "upvote_count" : int, - "url" : "https://imgur.com/a/TcBmP", - "view_count" : int, - "virality" : int, - }, - "account_id" : 0, - "count" : 19, - "date" : "type:datetime", - "description": "", - "ext" : "jpg", - "has_sound" : False, - "height" : int, - "id" : str, - "is_animated": False, - "is_looping" : False, - "mime_type" : "image/jpeg", - "name" : str, - "num" : int, - "size" : int, - "title" : str, - "type" : "image", - "updated_at" : None, - "url" : str, - "width" : int, - }, - }), - ("https://imgur.com/a/eD9CT", { # large album - "url": "de748c181a04d18bef1de9d4f4866ef0a06d632b", - }), - ("https://imgur.com/a/RhJXhVT/all", { # 7 character album hash - "url": "695ef0c950023362a0163ee5041796300db76674", - }), - ("https://imgur.com/a/TcBmQ", { - "exception": exception.HttpError, - }), - ("https://imgur.com/a/pjOnJA0", { # empty, no 'media' (#2557) - "count": 0, - }), - ("https://www.imgur.com/a/TcBmP"), # www - ("https://imgur.io/a/TcBmP"), # .io - ("https://m.imgur.com/a/TcBmP"), # mobile - ) + example = "https://imgur.com/a/abcde" def items(self): album = self.api.album(self.key) @@ -249,17 +122,7 @@ class ImgurGalleryExtractor(ImgurExtractor): """Extractor for imgur galleries""" subcategory = "gallery" pattern = BASE_PATTERN + r"/(?:gallery|t/\w+)/(\w{7}|\w{5})" - test = ( - ("https://imgur.com/gallery/zf2fIms", { # non-album gallery (#380) - "pattern": "https://imgur.com/zf2fIms", - }), - ("https://imgur.com/gallery/eD9CT", { - "pattern": "https://imgur.com/a/eD9CT", - }), - ("https://imgur.com/t/unmuted/26sEhNr"), - ("https://imgur.com/t/cat/qSB8NbN"), - ("https://imgur.io/t/cat/qSB8NbN"), # .io - ) + example = "https://imgur.com/gallery/abcde" def items(self): if self.api.gallery(self.key)["is_album"]: @@ -275,15 +138,7 @@ class ImgurUserExtractor(ImgurExtractor): """Extractor for all images posted by a user""" subcategory = "user" pattern = BASE_PATTERN + r"/user/([^/?#]+)(?:/posts|/submitted)?/?$" - test = ( - ("https://imgur.com/user/Miguenzo", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }), - ("https://imgur.com/user/Miguenzo/posts"), - ("https://imgur.com/user/Miguenzo/submitted"), - ) + example = "https://imgur.com/user/USER" def items(self): return self._items_queue(self.api.account_submissions(self.key)) @@ -293,11 +148,7 @@ class ImgurFavoriteExtractor(ImgurExtractor): """Extractor for a user's favorites""" subcategory = "favorite" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/?$" - test = ("https://imgur.com/user/Miguenzo/favorites", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/user/USER/favorites" def items(self): return self._items_queue(self.api.account_favorites(self.key)) @@ -307,16 +158,7 @@ class ImgurFavoriteFolderExtractor(ImgurExtractor): """Extractor for a user's favorites folder""" subcategory = "favorite-folder" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/folder/(\d+)" - test = ( - ("https://imgur.com/user/mikf1/favorites/folder/11896757/public", { - "pattern": r"https://imgur\.com(/a)?/\w+$", - "count": 3, - }), - ("https://imgur.com/user/mikf1/favorites/folder/11896741/private", { - "pattern": r"https://imgur\.com(/a)?/\w+$", - "count": 5, - }), - ) + example = "https://imgur.com/user/USER/favorites/folder/12345/TITLE" def __init__(self, match): ImgurExtractor.__init__(self, match) @@ -331,11 +173,7 @@ class ImgurSubredditExtractor(ImgurExtractor): """Extractor for a subreddits's imgur links""" subcategory = "subreddit" pattern = BASE_PATTERN + r"/r/([^/?#]+)/?$" - test = ("https://imgur.com/r/pics", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/r/SUBREDDIT" def items(self): return self._items_queue(self.api.gallery_subreddit(self.key)) @@ -345,11 +183,7 @@ class ImgurTagExtractor(ImgurExtractor): """Extractor for imgur tag searches""" subcategory = "tag" pattern = BASE_PATTERN + r"/t/([^/?#]+)$" - test = ("https://imgur.com/t/animals", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/t/TAG" def items(self): return self._items_queue(self.api.gallery_tag(self.key)) @@ -359,11 +193,7 @@ class ImgurSearchExtractor(ImgurExtractor): """Extractor for imgur search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/[^?#]+)?/?\?q=([^&#]+)" - test = ("https://imgur.com/search?q=cute+cat", { - "range": "1-100", - "count": 100, - "pattern": r"https://imgur\.com(/a)?/\w+$", - }) + example = "https://imgur.com/search?q=UERY" def items(self): key = text.unquote(self.key.replace("+", " ")) @@ -451,11 +281,7 @@ class ImgurAPI(): params["client_id"] = self.client_id params["page"] = 0 params["sort"] = "newest" - - headers = { - "Referer": "https://imgur.com/", - "Origin": "https://imgur.com", - } + headers = {"Origin": "https://imgur.com"} while True: data = self._call(endpoint, params, headers)["data"] diff --git a/gallery_dl/extractor/inkbunny.py b/gallery_dl/extractor/inkbunny.py index c6df16b0..4ad37fc1 100644 --- a/gallery_dl/extractor/inkbunny.py +++ b/gallery_dl/extractor/inkbunny.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -72,51 +72,7 @@ class InkbunnyUserExtractor(InkbunnyExtractor): """Extractor for inkbunny user profiles""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!s/)(gallery/|scraps/)?(\w+)(?:$|[/?#])" - test = ( - ("https://inkbunny.net/soina", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/\d+/\d+_soina_.+", - "range": "20-50", - "keyword": { - "date" : "type:datetime", - "deleted" : bool, - "file_id" : "re:[0-9]+", - "filename" : r"re:[0-9]+_soina_\w+", - "full_file_md5": "re:[0-9a-f]{32}", - "mimetype" : str, - "submission_id": "re:[0-9]+", - "user_id" : "20969", - "comments_count" : "re:[0-9]+", - "deleted" : bool, - "favorite" : bool, - "favorites_count": "re:[0-9]+", - "friends_only" : bool, - "guest_block" : bool, - "hidden" : bool, - "pagecount" : "re:[0-9]+", - "pools" : list, - "pools_count" : int, - "public" : bool, - "rating_id" : "re:[0-9]+", - "rating_name" : str, - "ratings" : list, - "scraps" : bool, - "tags" : list, - "title" : str, - "type_name" : str, - "username" : "soina", - "views" : str, - }, - }), - ("https://inkbunny.net/gallery/soina", { - "range": "1-25", - "keyword": {"scraps": False}, - }), - ("https://inkbunny.net/scraps/soina", { - "range": "1-25", - "keyword": {"scraps": True}, - }), - ) + example = "https://inkbunny.net/USER" def __init__(self, match): kind, self.user = match.groups() @@ -148,14 +104,7 @@ class InkbunnyPoolExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"poolview_process\.php\?pool_id=(\d+)|" r"submissionsviewall\.php\?([^#]+&mode=pool&[^#]+))") - test = ( - ("https://inkbunny.net/poolview_process.php?pool_id=28985", { - "count": 9, - "keyword": {"pool_id": "28985"}, - }), - ("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no"), - ) + example = "https://inkbunny.net/poolview_process.php?pool_id=12345" def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -185,16 +134,8 @@ class InkbunnyFavoriteExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"userfavorites_process\.php\?favs_user_id=(\d+)|" r"submissionsviewall\.php\?([^#]+&mode=userfavs&[^#]+))") - test = ( - ("https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/\d+/\d+_\w+_.+", - "range": "20-50", - "keyword": {"favs_user_id": "20969"}, - }), - ("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969"), - ) + example = ("https://inkbunny.net/userfavorites_process.php" + "?favs_user_id=12345") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -225,26 +166,8 @@ class InkbunnySearchExtractor(InkbunnyExtractor): subcategory = "search" pattern = (BASE_PATTERN + r"/submissionsviewall\.php\?([^#]+&mode=search&[^#]+)") - test = (("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" - "&mode=search&page=1&orderby=create_datetime&text=cute" - "&stringtype=and&keywords=yes&title=yes&description=no&artist=" - "&favsby=&type=&days=&keyword_id=&user_id=&random=&md5="), { - "range": "1-10", - "count": 10, - "keyword": { - "search": { - "rid": "ffffffffff", - "mode": "search", - "page": "1", - "orderby": "create_datetime", - "text": "cute", - "stringtype": "and", - "keywords": "yes", - "title": "yes", - "description": "no", - }, - }, - }) + example = ("https://inkbunny.net/submissionsviewall.php" + "?text=TAG&mode=search&type=") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -279,15 +202,8 @@ class InkbunnyFollowingExtractor(InkbunnyExtractor): pattern = (BASE_PATTERN + r"/(?:" r"watchlist_process\.php\?mode=watching&user_id=(\d+)|" r"usersviewall\.php\?([^#]+&mode=watching&[^#]+))") - test = ( - (("https://inkbunny.net/watchlist_process.php" - "?mode=watching&user_id=20969"), { - "pattern": InkbunnyUserExtractor.pattern, - "count": ">= 90", - }), - ("https://inkbunny.net/usersviewall.php?rid=ffffffffff" - "&mode=watching&page=1&user_id=20969&orderby=added&namesonly="), - ) + example = ("https://inkbunny.net/watchlist_process.php" + "?mode=watching&user_id=12345") def __init__(self, match): InkbunnyExtractor.__init__(self, match) @@ -324,16 +240,7 @@ class InkbunnyPostExtractor(InkbunnyExtractor): """Extractor for individual Inkbunny posts""" subcategory = "post" pattern = BASE_PATTERN + r"/s/(\d+)" - test = ( - ("https://inkbunny.net/s/1829715", { - "pattern": r"https://[\w.]+\.metapix\.net/files/full" - r"/2626/2626843_soina_dscn2296\.jpg", - "content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", - }), - ("https://inkbunny.net/s/2044094", { - "count": 4, - }), - ) + example = "https://inkbunny.net/s/12345" def __init__(self, match): InkbunnyExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index a00d73f8..e9631909 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -90,7 +90,9 @@ class InstagramExtractor(Extractor): file["_http_headers"] = video_headers text.nameext_from_url(url, file) yield Message.Url, url, file - if not previews: + if previews: + file["media_id"] += "p" + else: continue url = file["display_url"] @@ -396,11 +398,7 @@ class InstagramUserExtractor(InstagramExtractor): """Extractor for an Instagram user profile""" subcategory = "user" pattern = USER_PATTERN + r"/?(?:$|[?#])" - test = ( - ("https://www.instagram.com/instagram/"), - ("https://www.instagram.com/instagram/?hl=en"), - ("https://www.instagram.com/id:25025320/"), - ) + example = "https://www.instagram.com/USER/" def initialize(self): pass @@ -425,10 +423,7 @@ class InstagramPostsExtractor(InstagramExtractor): """Extractor for an Instagram user's posts""" subcategory = "posts" pattern = USER_PATTERN + r"/posts" - test = ("https://www.instagram.com/instagram/posts/", { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/USER/posts/" def posts(self): uid = self.api.user_id(self.item) @@ -439,10 +434,7 @@ class InstagramReelsExtractor(InstagramExtractor): """Extractor for an Instagram user's reels""" subcategory = "reels" pattern = USER_PATTERN + r"/reels" - test = ("https://www.instagram.com/instagram/reels/", { - "range": "40-60", - "count": ">= 20", - }) + example = "https://www.instagram.com/USER/reels/" def posts(self): uid = self.api.user_id(self.item) @@ -453,15 +445,7 @@ class InstagramTaggedExtractor(InstagramExtractor): """Extractor for an Instagram user's tagged posts""" subcategory = "tagged" pattern = USER_PATTERN + r"/tagged" - test = ("https://www.instagram.com/instagram/tagged/", { - "range": "1-16", - "count": ">= 16", - "keyword": { - "tagged_owner_id" : "25025320", - "tagged_username" : "instagram", - "tagged_full_name": "Instagram", - }, - }) + example = "https://www.instagram.com/USER/tagged/" def metadata(self): if self.item.startswith("id:"): @@ -485,11 +469,7 @@ class InstagramGuideExtractor(InstagramExtractor): """Extractor for an Instagram guide""" subcategory = "guide" pattern = USER_PATTERN + r"/guide/[^/?#]+/(\d+)" - test = (("https://www.instagram.com/kadakaofficial/guide" - "/knit-i-need-collection/18131821684305217/"), { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/USER/guide/NAME/12345" def __init__(self, match): InstagramExtractor.__init__(self, match) @@ -506,10 +486,7 @@ class InstagramSavedExtractor(InstagramExtractor): """Extractor for an Instagram user's saved media""" subcategory = "saved" pattern = USER_PATTERN + r"/saved(?:/all-posts)?/?$" - test = ( - ("https://www.instagram.com/instagram/saved/"), - ("https://www.instagram.com/instagram/saved/all-posts/"), - ) + example = "https://www.instagram.com/USER/saved/" def posts(self): return self.api.user_saved() @@ -519,9 +496,7 @@ class InstagramCollectionExtractor(InstagramExtractor): """Extractor for Instagram collection""" subcategory = "collection" pattern = USER_PATTERN + r"/saved/([^/?#]+)/([^/?#]+)" - test = ( - "https://www.instagram.com/instagram/saved/collection_name/123456789/", - ) + example = "https://www.instagram.com/USER/saved/COLLECTION/12345" def __init__(self, match): InstagramExtractor.__init__(self, match) @@ -543,14 +518,7 @@ class InstagramStoriesExtractor(InstagramExtractor): pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" r"/s(?:tories/(?:highlights/(\d+)|([^/?#]+)(?:/(\d+))?)" r"|/(aGlnaGxpZ2h0[^?#]+)(?:\?story_media_id=(\d+))?)") - test = ( - ("https://www.instagram.com/stories/instagram/"), - ("https://www.instagram.com/stories/highlights/18042509488170095/"), - ("https://instagram.com/stories/geekmig/2724343156064789461"), - ("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1"), - ("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1" - "?story_media_id=2724343156064789461"), - ) + example = "https://www.instagram.com/stories/USER/" def __init__(self, match): h1, self.user, m1, h2, m2 = match.groups() @@ -585,22 +553,33 @@ class InstagramHighlightsExtractor(InstagramExtractor): """Extractor for an Instagram user's story highlights""" subcategory = "highlights" pattern = USER_PATTERN + r"/highlights" - test = ("https://www.instagram.com/instagram/highlights",) + example = "https://www.instagram.com/USER/highlights/" def posts(self): uid = self.api.user_id(self.item) return self.api.highlights_media(uid) +class InstagramFollowingExtractor(InstagramExtractor): + """Extractor for an Instagram user's followed users""" + subcategory = "following" + pattern = USER_PATTERN + r"/following" + example = "https://www.instagram.com/USER/following/" + + def items(self): + uid = self.api.user_id(self.item) + for user in self.api.user_following(uid): + user["_extractor"] = InstagramUserExtractor + url = "{}/{}".format(self.root, user["username"]) + yield Message.Queue, url, user + + class InstagramTagExtractor(InstagramExtractor): """Extractor for Instagram tags""" subcategory = "tag" directory_fmt = ("{category}", "{subcategory}", "{tag}") pattern = BASE_PATTERN + r"/explore/tags/([^/?#]+)" - test = ("https://www.instagram.com/explore/tags/instagram/", { - "range": "1-16", - "count": ">= 16", - }) + example = "https://www.instagram.com/explore/tags/TAG/" def metadata(self): return {"tag": text.unquote(self.item)} @@ -613,10 +592,7 @@ class InstagramAvatarExtractor(InstagramExtractor): """Extractor for an Instagram user's avatar""" subcategory = "avatar" pattern = USER_PATTERN + r"/avatar" - test = ("https://www.instagram.com/instagram/avatar", { - "pattern": r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19" - r"/281440578_1088265838702675_6233856337905829714_n\.jpg", - }) + example = "https://www.instagram.com/USER/avatar/" def posts(self): if self._logged_in: @@ -656,102 +632,7 @@ class InstagramPostExtractor(InstagramExtractor): subcategory = "post" pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" r"/(?:[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)") - test = ( - # GraphImage - ("https://www.instagram.com/p/BqvsDleB3lV/", { - "pattern": r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)" - r"/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35" - r"/44877605_725955034447492_3123079845831750529_n.jpg", - "keyword": { - "date": "dt:2018-11-29 01:04:04", - "description": str, - "height": int, - "likes": int, - "location_id": "214424288", - "location_slug": "hong-kong", - "location_url": "re:/explore/locations/214424288/hong-kong/", - "media_id": "1922949326347663701", - "shortcode": "BqvsDleB3lV", - "post_id": "1922949326347663701", - "post_shortcode": "BqvsDleB3lV", - "post_url": "https://www.instagram.com/p/BqvsDleB3lV/", - "tags": ["#WHPsquares"], - "typename": "GraphImage", - "username": "instagram", - "width": int, - } - }), - # GraphSidecar - ("https://www.instagram.com/p/BoHk1haB5tM/", { - "count": 5, - "keyword": { - "sidecar_media_id": "1875629777499953996", - "sidecar_shortcode": "BoHk1haB5tM", - "post_id": "1875629777499953996", - "post_shortcode": "BoHk1haB5tM", - "post_url": "https://www.instagram.com/p/BoHk1haB5tM/", - "num": int, - "likes": int, - "username": "instagram", - } - }), - # GraphVideo - ("https://www.instagram.com/p/Bqxp0VSBgJg/", { - "pattern": r"/46840863_726311431074534_7805566102611403091_n\.mp4", - "keyword": { - "date": "dt:2018-11-29 19:23:58", - "description": str, - "height": int, - "likes": int, - "media_id": "1923502432034620000", - "post_url": "https://www.instagram.com/p/Bqxp0VSBgJg/", - "shortcode": "Bqxp0VSBgJg", - "tags": ["#ASMR"], - "typename": "GraphVideo", - "username": "instagram", - "width": int, - } - }), - # GraphVideo (IGTV) - ("https://www.instagram.com/tv/BkQjCfsBIzi/", { - "pattern": r"/10000000_597132547321814_702169244961988209_n\.mp4", - "keyword": { - "date": "dt:2018-06-20 19:51:32", - "description": str, - "height": int, - "likes": int, - "media_id": "1806097553666903266", - "post_url": "https://www.instagram.com/p/BkQjCfsBIzi/", - "shortcode": "BkQjCfsBIzi", - "typename": "GraphVideo", - "username": "instagram", - "width": int, - } - }), - # GraphSidecar with 2 embedded GraphVideo objects - ("https://www.instagram.com/p/BtOvDOfhvRr/", { - "count": 2, - "keyword": { - "post_url": "https://www.instagram.com/p/BtOvDOfhvRr/", - "sidecar_media_id": "1967717017113261163", - "sidecar_shortcode": "BtOvDOfhvRr", - "video_url": str, - } - }), - # GraphImage with tagged user - ("https://www.instagram.com/p/B_2lf3qAd3y/", { - "keyword": { - "tagged_users": [{ - "id" : "1246468638", - "username" : "kaaymbl", - "full_name": "Call Me Kay", - }] - } - }), - # URL with username (#2085) - ("https://www.instagram.com/dm/p/CW042g7B9CY/"), - ("https://www.instagram.com/reel/CDg_6Y1pxWu/"), - ) + example = "https://www.instagram.com/p/abcdefg/" def posts(self): return self.api.media(self.item) @@ -869,6 +750,11 @@ class InstagramRestAPI(): params = {"count": 30} return self._pagination(endpoint, params) + def user_following(self, user_id): + endpoint = "/v1/friendships/{}/following/".format(user_id) + params = {"count": 12} + return self._pagination_following(endpoint, params) + def user_saved(self): endpoint = "/v1/feed/saved/posts/" params = {"count": 50} @@ -958,6 +844,20 @@ class InstagramRestAPI(): return extr._update_cursor(None) params["max_id"] = extr._update_cursor(data["next_max_id"]) + def _pagination_following(self, endpoint, params): + extr = self.extractor + params["max_id"] = text.parse_int(extr._init_cursor()) + + while True: + data = self._call(endpoint, params=params) + + yield from data["users"] + + if len(data["users"]) < params["count"]: + return extr._update_cursor(None) + params["max_id"] = extr._update_cursor( + params["max_id"] + params["count"]) + class InstagramGraphqlAPI(): diff --git a/gallery_dl/extractor/issuu.py b/gallery_dl/extractor/issuu.py index 558c4fd7..f6170c28 100644 --- a/gallery_dl/extractor/issuu.py +++ b/gallery_dl/extractor/issuu.py @@ -26,31 +26,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor): filename_fmt = "{num:>03}.{extension}" archive_fmt = "{document[publicationId]}_{num}" pattern = r"(?:https?://)?issuu\.com(/[^/?#]+/docs/[^/?#]+)" - test = ("https://issuu.com/issuu/docs/motions-1-2019/", { - "pattern": r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", - "count" : 36, - "keyword": { - "document": { - "access" : "PUBLIC", - "contentRating" : { - "isAdsafe" : True, - "isExplicit": False, - "isReviewed": True, - }, - "date" : "dt:2019-09-16 00:00:00", - "description" : "re:Motions, the brand new publication by I", - "documentName" : "motions-1-2019", - "downloadable" : False, - "pageCount" : 36, - "publicationId" : "d99ec95935f15091b040cb8060f05510", - "title" : "Motions by Issuu - Issue 1", - "username" : "issuu", - }, - "extension": "jpg", - "filename" : r"re:page_\d+", - "num" : int, - }, - }) + example = "https://issuu.com/issuu/docs/TITLE/" def metadata(self, page): data = util.json_loads(text.rextract( @@ -78,10 +54,7 @@ class IssuuUserExtractor(IssuuBase, Extractor): """Extractor for all publications of a user/publisher""" subcategory = "user" pattern = r"(?:https?://)?issuu\.com/([^/?#]+)/?$" - test = ("https://issuu.com/issuu", { - "pattern": IssuuPublicationExtractor.pattern, - "count" : "> 25", - }) + example = "https://issuu.com/USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/itaku.py b/gallery_dl/extractor/itaku.py index 356a002b..1aef66e9 100644 --- a/gallery_dl/extractor/itaku.py +++ b/gallery_dl/extractor/itaku.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -63,12 +63,7 @@ class ItakuGalleryExtractor(ItakuExtractor): """Extractor for posts from an itaku user gallery""" subcategory = "gallery" pattern = BASE_PATTERN + r"/profile/([^/?#]+)/gallery" - test = ("https://itaku.ee/profile/piku/gallery", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs" - r"/[^/?#]+\.(jpg|png|gif)", - "range": "1-10", - "count": 10, - }) + example = "https://itaku.ee/profile/USER/gallery" def posts(self): return self.api.galleries_images(self.item) @@ -77,62 +72,7 @@ class ItakuGalleryExtractor(ItakuExtractor): class ItakuImageExtractor(ItakuExtractor): subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" - test = ( - ("https://itaku.ee/images/100471", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs" - r"/220504_oUNIAFT\.png", - "count": 1, - "keyword": { - "already_pinned": None, - "blacklisted": { - "blacklisted_tags": [], - "is_blacklisted": False - }, - "can_reshare": True, - "date": "dt:2022-05-05 19:21:17", - "date_added": "2022-05-05T19:21:17.674148Z", - "date_edited": "2022-05-25T14:37:46.220612Z", - "description": "sketch from drawpile", - "extension": "png", - "filename": "220504_oUNIAFT", - "hotness_score": float, - "id": 100471, - "image": "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs" - "/220504_oUNIAFT.png", - "image_xl": "https://d1wmr8tlk3viaj.cloudfront.net" - "/gallery_imgs/220504_oUNIAFT/lg.jpg", - "liked_by_you": False, - "maturity_rating": "SFW", - "num_comments": int, - "num_likes": int, - "num_reshares": int, - "obj_tags": 136446, - "owner": 16775, - "owner_avatar": "https://d1wmr8tlk3viaj.cloudfront.net" - "/profile_pics/av2022r_vKYVywc/md.jpg", - "owner_displayname": "Piku", - "owner_username": "piku", - "reshared_by_you": False, - "sections": ["Fanart/Miku"], - "tags": list, - "tags_character": ["hatsune_miku"], - "tags_copyright": ["vocaloid"], - "tags_general" : ["twintails", "green_hair", "flag", - "gloves", "green_eyes", "female", - "racing_miku"], - "title": "Racing Miku 2022 Ver.", - "too_mature": False, - "uncompressed_filesize": "0.62", - "video": None, - "visibility": "PUBLIC", - }, - }), - # video - ("https://itaku.ee/images/19465", { - "pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids" - r"/sleepy_af_OY5GHWw\.mp4", - }), - ) + example = "https://itaku.ee/images/12345" def posts(self): return (self.api.image(self.item),) @@ -145,7 +85,6 @@ class ItakuAPI(): self.root = extractor.root + "/api" self.headers = { "Accept": "application/json, text/plain, */*", - "Referer": extractor.root + "/", } def galleries_images(self, username, section=None): diff --git a/gallery_dl/extractor/itchio.py b/gallery_dl/extractor/itchio.py index 96ebbdc4..799dd66e 100644 --- a/gallery_dl/extractor/itchio.py +++ b/gallery_dl/extractor/itchio.py @@ -21,28 +21,7 @@ class ItchioGameExtractor(Extractor): filename_fmt = "{game[title]} ({id}).{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?(\w+).itch\.io/([\w-]+)" - test = ( - ("https://sirtartarus.itch.io/a-craft-of-mine", { - "pattern": r"https://\w+\.ssl\.hwcdn\.net/upload2" - r"/game/1983311/7723751\?", - "count": 1, - "keyword": { - "extension": "", - "filename": "7723751", - "game": { - "id": 1983311, - "noun": "game", - "title": "A Craft Of Mine", - "url": "https://sirtartarus.itch.io/a-craft-of-mine", - }, - "user": { - "id": 4060052, - "name": "SirTartarus", - "url": "https://sirtartarus.itch.io", - }, - }, - }), - ) + example = "https://USER.itch.io/GAME" def __init__(self, match): self.user, self.slug = match.groups() diff --git a/gallery_dl/extractor/jpgfish.py b/gallery_dl/extractor/jpgfish.py index 39208e5a..8862a7b7 100644 --- a/gallery_dl/extractor/jpgfish.py +++ b/gallery_dl/extractor/jpgfish.py @@ -4,18 +4,18 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for https://jpeg.pet/""" +"""Extractors for https://jpg1.su/""" from .common import Extractor, Message from .. import text -BASE_PATTERN = r"(?:https?://)?jpe?g\.(?:pet|fish(?:ing)?|church)" +BASE_PATTERN = r"(?:https?://)?jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)" class JpgfishExtractor(Extractor): """Base class for jpgfish extractors""" category = "jpgfish" - root = "https://jpeg.pet" + root = "https://jpg1.su" directory_fmt = ("{category}", "{user}", "{album}",) archive_fmt = "{id}" @@ -35,28 +35,7 @@ class JpgfishImageExtractor(JpgfishExtractor): """Extractor for jpgfish Images""" subcategory = "image" pattern = BASE_PATTERN + r"/img/((?:[^/?#]+\.)?(\w+))" - test = ( - ("https://jpeg.pet/img/funnymeme.LecXGS", { - "pattern": r"https://simp3\.jpg\.church/images/funnymeme\.jpg", - "content": "098e5e9b17ad634358426e0ffd1c93871474d13c", - "keyword": { - "album": "", - "extension": "jpg", - "filename": "funnymeme", - "id": "LecXGS", - "url": "https://simp3.jpg.church/images/funnymeme.jpg", - "user": "exearco", - }, - }), - ("https://jpg.church/img/auCruA", { - "pattern": r"https://simp2\.jpg\.church/hannahowo_00457\.jpg", - "keyword": {"album": "401-500"}, - }), - ("https://jpg.pet/img/funnymeme.LecXGS"), - ("https://jpg.fishing/img/funnymeme.LecXGS"), - ("https://jpg.fish/img/funnymeme.LecXGS"), - ("https://jpg.church/img/funnymeme.LecXGS"), - ) + example = "https://jpg1.su/img/TITLE.ID" def __init__(self, match): JpgfishExtractor.__init__(self, match) @@ -83,21 +62,7 @@ class JpgfishAlbumExtractor(JpgfishExtractor): """Extractor for jpgfish Albums""" subcategory = "album" pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?" - test = ( - ("https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", { - "count": 2, - }), - ("https://jpg.fishing/a/gunggingnsk.N9OOI", { - "count": 114, - }), - ("https://jpg.fish/a/101-200.aNJ6A/", { - "count": 100, - }), - ("https://jpg.church/a/hannahowo.aNTdH/sub", { - "count": 606, - }), - ("https://jpg.pet/album/CDilP/?sort=date_desc&page=1"), - ) + example = "https://jpg1.su/album/TITLE.ID" def __init__(self, match): JpgfishExtractor.__init__(self, match) @@ -121,18 +86,7 @@ class JpgfishUserExtractor(JpgfishExtractor): """Extractor for jpgfish Users""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?" - test = ( - ("https://jpeg.pet/exearco", { - "count": 3, - }), - ("https://jpg.church/exearco/albums", { - "count": 1, - }), - ("https://jpg.pet/exearco"), - ("https://jpg.fishing/exearco"), - ("https://jpg.fish/exearco"), - ("https://jpg.church/exearco"), - ) + example = "https://jpg1.su/USER" def __init__(self, match): JpgfishExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/jschan.py b/gallery_dl/extractor/jschan.py index fe758faa..398256dc 100644 --- a/gallery_dl/extractor/jschan.py +++ b/gallery_dl/extractor/jschan.py @@ -31,12 +31,7 @@ class JschanThreadExtractor(JschanExtractor): filename_fmt = "{postId}{num:?-//} {filename}.{extension}" archive_fmt = "{board}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)\.html" - test = ( - ("https://94chan.org/art/thread/25.html", { - "pattern": r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", - "count": ">= 15" - }) - ) + example = "https://94chan.org/a/thread/12345.html" def __init__(self, match): JschanExtractor.__init__(self, match) @@ -71,15 +66,7 @@ class JschanBoardExtractor(JschanExtractor): subcategory = "board" pattern = (BASE_PATTERN + r"/([^/?#]+)" r"(?:/index\.html|/catalog\.html|/\d+\.html|/?$)") - test = ( - ("https://94chan.org/art/", { - "pattern": JschanThreadExtractor.pattern, - "count": ">= 30" - }), - ("https://94chan.org/art/2.html"), - ("https://94chan.org/art/catalog.html"), - ("https://94chan.org/art/index.html"), - ) + example = "https://94chan.org/a/" def __init__(self, match): JschanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/kabeuchi.py b/gallery_dl/extractor/kabeuchi.py index f172dcfa..867f0da8 100644 --- a/gallery_dl/extractor/kabeuchi.py +++ b/gallery_dl/extractor/kabeuchi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,16 +21,7 @@ class KabeuchiUserExtractor(Extractor): archive_fmt = "{id}_{num}" root = "https://kabe-uchiroom.com" pattern = r"(?:https?://)?kabe-uchiroom\.com/mypage/?\?id=(\d+)" - test = ( - ("https://kabe-uchiroom.com/mypage/?id=919865303848255493", { - "pattern": (r"https://kabe-uchiroom\.com/accounts/upfile/3/" - r"919865303848255493/\w+\.jpe?g"), - "count": ">= 24", - }), - ("https://kabe-uchiroom.com/mypage/?id=123456789", { - "exception": exception.NotFoundError, - }), - ) + example = "https://kabe-uchiroom.com/mypage/?id=12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/keenspot.py b/gallery_dl/extractor/keenspot.py index b5d77380..b8ecc143 100644 --- a/gallery_dl/extractor/keenspot.py +++ b/gallery_dl/extractor/keenspot.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,28 +20,7 @@ class KeenspotComicExtractor(Extractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{comic}_{filename}" pattern = r"(?:https?://)?(?!www\.|forums\.)([\w-]+)\.keenspot\.com(/.+)?" - test = ( - ("http://marksmen.keenspot.com/", { # link - "range": "1-3", - "url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", - }), - ("http://barkercomic.keenspot.com/", { # id - "range": "1-3", - "url": "c4080926db18d00bac641fdd708393b7d61379e6", - }), - ("http://crowscare.keenspot.com/", { # id v2 - "range": "1-3", - "url": "a00e66a133dd39005777317da90cef921466fcaa" - }), - ("http://supernovas.keenspot.com/", { # ks - "range": "1-3", - "url": "de21b12887ef31ff82edccbc09d112e3885c3aab" - }), - ("http://twokinds.keenspot.com/comic/1066/", { # "random" access - "range": "1-3", - "url": "6a784e11370abfb343dcad9adbb7718f9b7be350", - }) - ) + example = "http://COMIC.keenspot.com/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 2ed73e9c..894c6719 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -37,7 +37,6 @@ class KemonopartyExtractor(Extractor): Extractor.__init__(self, match) def _init(self): - self.session.headers["Referer"] = self.root + "/" self._prepare_ddosguard_cookies() self._find_inline = re.compile( r'src="(?:https?://(?:kemono|coomer)\.(?:party|su))?(/inline/[^"]+' @@ -216,19 +215,7 @@ class KemonopartyUserExtractor(KemonopartyExtractor): """Extractor for all posts from a kemono.party user listing""" subcategory = "user" pattern = USER_PATTERN + r"/?(?:\?o=(\d+))?(?:$|[?#])" - test = ( - ("https://kemono.party/fanbox/user/6993449", { - "range": "1-25", - "count": 25, - }), - # 'max-posts' option, 'o' query parameter (#1674) - ("https://kemono.party/patreon/user/881792?o=150", { - "options": (("max-posts", 25),), - "count": "< 100", - }), - ("https://kemono.su/subscribestar/user/alcorart"), - ("https://kemono.party/subscribestar/user/alcorart"), - ) + example = "https://kemono.party/SERVICE/user/12345" def __init__(self, match): _, _, service, user_id, offset = match.groups() @@ -256,87 +243,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor): """Extractor for a single kemono.party post""" subcategory = "post" pattern = USER_PATTERN + r"/post/([^/?#]+)" - test = ( - ("https://kemono.party/fanbox/user/6993449/post/506575", { - "pattern": r"https://kemono.party/data/21/0f" - r"/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", - "content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", - "keyword": { - "added": "Wed, 06 May 2020 20:28:02 GMT", - "content": str, - "count": 1, - "date": "dt:2019-08-11 02:09:04", - "edited": None, - "embed": dict, - "extension": "jpeg", - "filename": "P058kDFYus7DbqAkGlfWTlOr", - "hash": "210f35388e28bbcf756db18dd516e2d8" - "2ce758e0d32881eeee76d43e1716d382", - "id": "506575", - "num": 1, - "published": "Sun, 11 Aug 2019 02:09:04 GMT", - "service": "fanbox", - "shared_file": False, - "subcategory": "fanbox", - "title": "c96取り置き", - "type": "file", - "user": "6993449", - }, - }), - # inline image (#1286) - ("https://kemono.party/fanbox/user/7356311/post/802343", { - "pattern": r"https://kemono\.party/data/47/b5/47b5c014ecdcfabdf2c8" - r"5eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", - "keyword": {"hash": "47b5c014ecdcfabdf2c85eec53f1133a" - "76336997ae8596f332e97d956a460ad2"}, - }), - # kemono.party -> data.kemono.party - ("https://kemono.party/gumroad/user/trylsc/post/IURjT", { - "pattern": r"https://kemono\.party/data/(" - r"a4/7b/a47bfe938d8c1682eef06e885927484cd8df1b.+\.jpg|" - r"c6/04/c6048f5067fd9dbfa7a8be565ac194efdfb6e4.+\.zip)", - }), - # username (#1548, #1652) - ("https://kemono.party/gumroad/user/3252870377455/post/aJnAH", { - "options": (("metadata", True),), - "keyword": {"username": "Kudalyn's Creations"}, - }), - # skip patreon duplicates - ("https://kemono.party/patreon/user/4158582/post/32099982", { - "count": 2, - }), - # allow duplicates (#2440) - ("https://kemono.party/patreon/user/4158582/post/32099982", { - "options": (("duplicates", True),), - "count": 3, - }), - # DMs (#2008) - ("https://kemono.party/patreon/user/34134344/post/38129255", { - "options": (("dms", True),), - "keyword": {"dms": [{ - "body": r"re:Hi! Thank you very much for supporting the work I" - r" did in May. Here's your reward pack! I hope you fin" - r"d something you enjoy in it. :\)\n\nhttps://www.medi" - r"afire.com/file/\w+/Set13_tier_2.zip/file", - "date": "2021-07-31 02:47:51.327865", - }]}, - }), - # coomer.party (#2100) - ("https://coomer.party/onlyfans/user/alinity/post/125962203", { - "pattern": r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968" - r"c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg", - }), - # invalid file (#3510) - ("https://kemono.party/patreon/user/19623797/post/29035449", { - "pattern": r"907ba78b4545338d3539683e63ecb51c" - r"f51c10adc9dabd86e92bd52339f298b9\.txt", - "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", # empty - }), - ("https://kemono.su/subscribestar/user/alcorart/post/184330"), - ("https://kemono.party/subscribestar/user/alcorart/post/184330"), - ("https://www.kemono.party/subscribestar/user/alcorart/post/184330"), - ("https://beta.kemono.party/subscribestar/user/alcorart/post/184330"), - ) + example = "https://kemono.party/SERVICE/user/12345/post/12345" def __init__(self, match): _, _, service, user_id, post_id = match.groups() @@ -359,30 +266,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): filename_fmt = "{id}_{num:>02}_{filename}.{extension}" archive_fmt = "discord_{server}_{id}_{num}" pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)" - test = ( - (("https://kemono.party/discord" - "/server/488668827274444803#finish-work"), { - "count": 4, - "keyword": {"channel_name": "finish-work"}, - }), - (("https://kemono.su/discord" - "/server/256559665620451329/channel/462437519519383555#"), { - "pattern": r"https://kemono\.su/data/(" - r"e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|" - r"51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", - "keyword": {"hash": "re:e377e3525164559484ace2e64425b0cec1db08" - "|51453640a5e0a4d23fbf57fb85390f9c5ec154"}, - "count": ">= 2", - }), - # 'inline' files - (("https://kemono.party/discord" - "/server/315262215055736843/channel/315262215055736843#general"), { - "pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", - "options": (("image-filter", "type == 'inline'"),), - "keyword": {"hash": ""}, - "range": "1-5", - }), - ) + example = "https://kemono.party/discord/server/12345#CHANNEL" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -461,16 +345,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): class KemonopartyDiscordServerExtractor(KemonopartyExtractor): subcategory = "discord-server" pattern = BASE_PATTERN + r"/discord/server/(\d+)$" - test = ( - ("https://kemono.party/discord/server/488668827274444803", { - "pattern": KemonopartyDiscordExtractor.pattern, - "count": 13, - }), - ("https://kemono.su/discord/server/488668827274444803", { - "pattern": KemonopartyDiscordExtractor.pattern, - "count": 13, - }), - ) + example = "https://kemono.party/discord/server/12345" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -492,23 +367,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): """Extractor for kemono.party favorites""" subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?" - test = ( - ("https://kemono.party/favorites", { - "pattern": KemonopartyUserExtractor.pattern, - "url": "f4b5b796979bcba824af84206578c79101c7f0e1", - "count": 3, - }), - ("https://kemono.party/favorites?type=post", { - "pattern": KemonopartyPostExtractor.pattern, - "url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f", - "count": 3, - }), - ("https://kemono.su/favorites?type=post", { - "pattern": KemonopartyPostExtractor.pattern, - "url": "4be8e84cb384a907a8e7997baaf6287b451783b5", - "count": 3, - }), - ) + example = "https://kemono.party/favorites" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -522,7 +381,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): if self.favorites == "artist": users = self.request( - self.root + "/api/favorites?type=artist").json() + self.root + "/api/v1/account/favorites?type=artist").json() for user in users: user["_extractor"] = KemonopartyUserExtractor url = "{}/{}/user/{}".format( @@ -531,7 +390,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor): elif self.favorites == "post": posts = self.request( - self.root + "/api/favorites?type=post").json() + self.root + "/api/v1/account/favorites?type=post").json() for post in posts: post["_extractor"] = KemonopartyPostExtractor url = "{}/{}/user/{}/post/{}".format( diff --git a/gallery_dl/extractor/khinsider.py b/gallery_dl/extractor/khinsider.py index 0c3b002f..d0c9c301 100644 --- a/gallery_dl/extractor/khinsider.py +++ b/gallery_dl/extractor/khinsider.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -16,30 +16,13 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor): """Extractor for soundtracks from khinsider.com""" category = "khinsider" subcategory = "soundtrack" + root = "https://downloads.khinsider.com" directory_fmt = ("{category}", "{album[name]}") archive_fmt = "{filename}.{extension}" pattern = (r"(?:https?://)?downloads\.khinsider\.com" r"/game-soundtracks/album/([^/?#]+)") - root = "https://downloads.khinsider.com" - test = (("https://downloads.khinsider.com" - "/game-soundtracks/album/horizon-riders-wii"), { - "pattern": r"https?://vgm(site|downloads)\.com" - r"/soundtracks/horizon-riders-wii/[^/]+" - r"/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", - "keyword": { - "album": { - "count": 1, - "date": "Sep 18th, 2016", - "name": "Horizon Riders", - "platform": "Wii", - "size": 26214400, - "type": "Gamerip", - }, - "extension": "mp3", - "filename": "Horizon Riders Wii - Full Soundtrack", - }, - "count": 1, - }) + example = ("https://downloads.khinsider.com" + "/game-soundtracks/album/TITLE") def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/komikcast.py b/gallery_dl/extractor/komikcast.py index 04373c4b..a3e01305 100644 --- a/gallery_dl/extractor/komikcast.py +++ b/gallery_dl/extractor/komikcast.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2022 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -48,18 +48,7 @@ class KomikcastBase(): class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor): """Extractor for manga-chapters from komikcast.site""" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/)" - test = ( - (("https://komikcast.site/chapter" - "/apotheosis-chapter-02-2-bahasa-indonesia/"), { - "url": "f6b43fbc027697749b3ea1c14931c83f878d7936", - "keyword": "f3938e1aff9ad1f302f52447e9781b21f6da26d4", - }), - (("https://komikcast.me/chapter" - "/soul-land-ii-chapter-300-1-bahasa-indonesia/"), { - "url": "efd00a9bd95461272d51990d7bc54b79ff3ff2e6", - "keyword": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", - }), - ) + example = "https://komikcast.site/chapter/TITLE/" def metadata(self, page): info = text.extr(page, "<title>", " - Komikcast<") @@ -79,13 +68,7 @@ class KomikcastMangaExtractor(KomikcastBase, MangaExtractor): """Extractor for manga from komikcast.site""" chapterclass = KomikcastChapterExtractor pattern = BASE_PATTERN + r"(/(?:komik/)?[^/?#]+)/?$" - test = ( - ("https://komikcast.site/komik/090-eko-to-issho/", { - "url": "19d3d50d532e84be6280a3d61ff0fd0ca04dd6b4", - "keyword": "837a7e96867344ff59d840771c04c20dc46c0ab1", - }), - ("https://komikcast.me/tonari-no-kashiwagi-san/"), - ) + example = "https://komikcast.site/komik/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/lensdump.py b/gallery_dl/extractor/lensdump.py index 43fc24e1..d4ccf33b 100644 --- a/gallery_dl/extractor/lensdump.py +++ b/gallery_dl/extractor/lensdump.py @@ -48,19 +48,7 @@ class LensdumpBase(): class LensdumpAlbumExtractor(LensdumpBase, GalleryExtractor): subcategory = "album" pattern = BASE_PATTERN + r"/(?:((?!\w+/albums|a/|i/)\w+)|a/(\w+))" - test = ( - ("https://lensdump.com/a/1IhJr", { - "pattern": r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", - "keyword": { - "extension": "png", - "name": str, - "num": int, - "title": str, - "url": str, - "width": int, - }, - }), - ) + example = "https://lensdump.com/a/ID" def __init__(self, match): GalleryExtractor.__init__(self, match, match.string) @@ -100,7 +88,7 @@ class LensdumpAlbumsExtractor(LensdumpBase, Extractor): """Extractor for album list from lensdump.com""" subcategory = "albums" pattern = BASE_PATTERN + r"/\w+/albums" - test = ("https://lensdump.com/vstar925/albums",) + example = "https://lensdump.com/USER/albums" def items(self): for node in self.nodes(): @@ -117,22 +105,7 @@ class LensdumpImageExtractor(LensdumpBase, Extractor): directory_fmt = ("{category}",) archive_fmt = "{id}" pattern = BASE_PATTERN + r"/i/(\w+)" - test = ( - ("https://lensdump.com/i/tyoAyM", { - "pattern": r"https://c\.l3n\.co/i/tyoAyM\.webp", - "content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", - "keyword": { - "date": "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename": "tyoAyM", - "height": 400, - "id": "tyoAyM", - "title": "MYOBI clovis bookcaseset", - "url": "https://c.l3n.co/i/tyoAyM.webp", - "width": 620, - }, - }), - ) + example = "https://lensdump.com/i/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/lexica.py b/gallery_dl/extractor/lexica.py index ad936257..d55d821e 100644 --- a/gallery_dl/extractor/lexica.py +++ b/gallery_dl/extractor/lexica.py @@ -20,37 +20,7 @@ class LexicaSearchExtractor(Extractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "{id}" pattern = r"(?:https?://)?lexica\.art/?\?q=([^&#]+)" - test = ( - ("https://lexica.art/?q=tree", { - "pattern": r"https://lexica-serve-encoded-images2\.sharif\." - r"workers.dev/full_jpg/[0-9a-f-]{36}$", - "range": "1-80", - "count": 80, - "keyword": { - "height": int, - "id": str, - "upscaled_height": int, - "upscaled_width": int, - "userid": str, - "width": int, - "prompt": { - "c": int, - "grid": bool, - "height": int, - "id": str, - "images": list, - "initImage": None, - "initImageStrength": None, - "model": "lexica-aperture-v2", - "negativePrompt": str, - "prompt": str, - "seed": str, - "timestamp": r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", - "width": int, - }, - }, - }), - ) + example = "https://lexica.art/?q=QUERY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/lightroom.py b/gallery_dl/extractor/lightroom.py index 783473d6..2cbaa976 100644 --- a/gallery_dl/extractor/lightroom.py +++ b/gallery_dl/extractor/lightroom.py @@ -18,24 +18,7 @@ class LightroomGalleryExtractor(Extractor): filename_fmt = "{num:>04}_{id}.{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?lightroom\.adobe\.com/shares/([0-9a-f]+)" - test = ( - (("https://lightroom.adobe.com/shares/" - "0c9cce2033f24d24975423fe616368bf"), { - "keyword": { - "title": "Sterne und Nachtphotos", - "user": "Christian Schrang", - }, - "count": ">= 55", - }), - (("https://lightroom.adobe.com/shares/" - "7ba68ad5a97e48608d2e6c57e6082813"), { - "keyword": { - "title": "HEBFC Snr/Res v Brighton", - "user": "", - }, - "count": ">= 180", - }), - ) + example = "https://lightroom.adobe.com/shares/0123456789abcdef" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/livedoor.py b/gallery_dl/extractor/livedoor.py index 2765f0b6..e21659f0 100644 --- a/gallery_dl/extractor/livedoor.py +++ b/gallery_dl/extractor/livedoor.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -84,32 +84,7 @@ class LivedoorBlogExtractor(LivedoorExtractor): """Extractor for a user's blog on blog.livedoor.jp""" subcategory = "blog" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/?(?:$|[?#])" - test = ( - ("http://blog.livedoor.jp/zatsu_ke/", { - "range": "1-50", - "count": 50, - "archive": False, - "pattern": r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", - "keyword": { - "post": { - "categories" : tuple, - "date" : "type:datetime", - "description": str, - "id" : int, - "tags" : list, - "title" : str, - "user" : "zatsu_ke" - }, - "filename": str, - "hash" : r"re:\w{4,}", - "num" : int, - }, - }), - ("http://blog.livedoor.jp/uotapo/", { - "range": "1-5", - "count": 5, - }), - ) + example = "http://blog.livedoor.jp/USER/" def posts(self): url = "{}/{}".format(self.root, self.user) @@ -129,20 +104,7 @@ class LivedoorPostExtractor(LivedoorExtractor): """Extractor for images from a blog post on blog.livedoor.jp""" subcategory = "post" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/archives/(\d+)" - test = ( - ("http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", { - "url": "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", - "keyword": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", - }), - ("http://blog.livedoor.jp/amaumauma/archives/7835811.html", { - "url": "204bbd6a9db4969c50e0923855aeede04f2e4a62", - "keyword": "05821c7141360e6057ef2d382b046f28326a799d", - }), - ("http://blog.livedoor.jp/uotapo/archives/1050616939.html", { - "url": "4b5ab144b7309eb870d9c08f8853d1abee9946d2", - "keyword": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", - }), - ) + example = "http://blog.livedoor.jp/USER/archives/12345.html" def __init__(self, match): LivedoorExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/lolisafe.py b/gallery_dl/extractor/lolisafe.py index 9cebe3ae..3d7d685b 100644 --- a/gallery_dl/extractor/lolisafe.py +++ b/gallery_dl/extractor/lolisafe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,17 +30,7 @@ BASE_PATTERN = LolisafeExtractor.update({ class LolisafeAlbumExtractor(LolisafeExtractor): subcategory = "album" pattern = BASE_PATTERN + "/a/([^/?#]+)" - test = ( - ("https://xbunkr.com/a/TA0bu3F4", { - "pattern": r"https://media\.xbunkr\.com/[^.]+\.\w+", - "count": 861, - "keyword": { - "album_id": "TA0bu3F4", - "album_name": "Hannahowo Onlyfans Photos", - } - }), - ("https://xbunkr.com/a/GNQc2I5d"), - ) + example = "https://xbunkr.com/a/ID" def __init__(self, match): LolisafeExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/luscious.py b/gallery_dl/extractor/luscious.py index dcf09d16..c3c44d2c 100644 --- a/gallery_dl/extractor/luscious.py +++ b/gallery_dl/extractor/luscious.py @@ -47,73 +47,7 @@ class LusciousAlbumExtractor(LusciousExtractor): archive_fmt = "{album[id]}_{id}" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)") - test = ( - ("https://luscious.net/albums/okinami-no-koigokoro_277031/", { - "pattern": r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+" - r"/images/NTRshouldbeillegal/277031" - r"/luscious_net_\d+_\d+\.jpg$", - # "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3", - "keyword": { - "album": { - "__typename" : "Album", - "audiences" : list, - "content" : "Hentai", - "cover" : "re:https://\\w+.luscious.net/.+/277031/", - "created" : 1479625853, - "created_by" : "NTRshouldbeillegal", - "date" : "dt:2016-11-20 07:10:53", - "description" : "Enjoy.", - "download_url": "re:/download/(r/)?824778/277031/", - "genres" : list, - "id" : 277031, - "is_manga" : True, - "labels" : list, - "language" : "English", - "like_status" : "none", - "modified" : int, - "permissions" : list, - "rating" : float, - "slug" : "okinami-no-koigokoro", - "status" : None, - "tags" : list, - "title" : "Okinami no Koigokoro", - "url" : "/albums/okinami-no-koigokoro_277031/", - "marked_for_deletion": False, - "marked_for_processing": False, - "number_of_animated_pictures": 0, - "number_of_favorites": int, - "number_of_pictures": 18, - }, - "aspect_ratio": r"re:\d+:\d+", - "category" : "luscious", - "created" : int, - "date" : "type:datetime", - "height" : int, - "id" : int, - "is_animated" : False, - "like_status" : "none", - "position" : int, - "resolution" : r"re:\d+x\d+", - "status" : None, - "tags" : list, - "thumbnail" : str, - "title" : str, - "width" : int, - "number_of_comments": int, - "number_of_favorites": int, - }, - }), - ("https://luscious.net/albums/not-found_277035/", { - "exception": exception.NotFoundError, - }), - ("https://members.luscious.net/albums/login-required_323871/", { - "count": 64, - }), - ("https://www.luscious.net/albums/okinami_277031/"), - ("https://members.luscious.net/albums/okinami_277031/"), - ("https://luscious.net/pictures/c/video_game_manga/album" - "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"), - ) + example = "https://luscious.net/albums/TITLE_12345/" def __init__(self, match): LusciousExtractor.__init__(self, match) @@ -338,15 +272,7 @@ class LusciousSearchExtractor(LusciousExtractor): subcategory = "search" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" r"/albums/list/?(?:\?([^#]+))?") - test = ( - ("https://members.luscious.net/albums/list/"), - ("https://members.luscious.net/albums/list/" - "?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", { - "pattern": LusciousAlbumExtractor.pattern, - "range": "41-60", - "count": 20, - }), - ) + example = "https://luscious.net/albums/list/?tagged=TAG" def __init__(self, match): LusciousExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/lynxchan.py b/gallery_dl/extractor/lynxchan.py index 85e8bb1e..0edd5c12 100644 --- a/gallery_dl/extractor/lynxchan.py +++ b/gallery_dl/extractor/lynxchan.py @@ -40,22 +40,7 @@ class LynxchanThreadExtractor(LynxchanExtractor): filename_fmt = "{postId}{num:?-//} {filename}.{extension}" archive_fmt = "{boardUri}_{postId}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://bbw-chan.nl/bbwdraw/res/499.html", { - "pattern": r"https://bbw-chan\.nl/\.media/[0-9a-f]{64}(\.\w+)?$", - "count": ">= 352", - }), - ("https://bbw-chan.nl/bbwdraw/res/489.html"), - ("https://kohlchan.net/a/res/4594.html", { - "pattern": r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", - "count": ">= 80", - }), - ("https://endchan.org/yuri/res/193483.html", { - "pattern": r"https://endchan\.org/\.media/[^.]+(\.\w+)?$", - "count" : ">= 19", - }), - ("https://endchan.org/yuri/res/33621.html"), - ) + example = "https://bbw-chan.nl/a/res/12345.html" def __init__(self, match): LynxchanExtractor.__init__(self, match) @@ -86,24 +71,7 @@ class LynxchanBoardExtractor(LynxchanExtractor): """Extractor for LynxChan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" - test = ( - ("https://bbw-chan.nl/bbwdraw/", { - "pattern": LynxchanThreadExtractor.pattern, - "count": ">= 148", - }), - ("https://bbw-chan.nl/bbwdraw/2.html"), - ("https://kohlchan.net/a/", { - "pattern": LynxchanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://kohlchan.net/a/2.html"), - ("https://kohlchan.net/a/catalog.html"), - ("https://endchan.org/yuri/", { - "pattern": LynxchanThreadExtractor.pattern, - "count" : ">= 9", - }), - ("https://endchan.org/yuri/catalog.html"), - ) + example = "https://bbw-chan.nl/a/" def __init__(self, match): LynxchanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index e12e56b4..dbaf4cb8 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -98,25 +98,8 @@ class MangadexChapterExtractor(MangadexExtractor): """Extractor for manga-chapters from mangadex.org""" subcategory = "chapter" pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)" - test = ( - ("https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", { - "keyword": "e86128a79ebe7201b648f1caa828496a2878dc8f", - # "content": "50383a4c15124682057b197d40261641a98db514", - }), - # oneshot - ("https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", { - "count": 64, - "keyword": "d11ed057a919854696853362be35fc0ba7dded4c", - }), - # MANGA Plus (#1154) - ("https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", { - "exception": exception.StopExtraction, - }), - # 'externalUrl', but still downloadable (#2503) - ("https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", { - "count": 0, # 404 - }), - ) + example = ("https://mangadex.org/chapter" + "/01234567-89ab-cdef-0123-456789abcdef") def items(self): try: @@ -148,48 +131,8 @@ class MangadexMangaExtractor(MangadexExtractor): """Extractor for manga from mangadex.org""" subcategory = "manga" pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)" - test = ( - ("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", { - "count": ">= 5", - "keyword": { - "manga" : "Souten no Koumori", - "manga_id": "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "title" : "re:One[Ss]hot", - "volume" : 0, - "chapter" : 0, - "chapter_minor": "", - "chapter_id": str, - "date" : "type:datetime", - "lang" : str, - "language": str, - "artist" : ["Arakawa Hiromu"], - "author" : ["Arakawa Hiromu"], - "status" : "completed", - "tags" : ["Oneshot", "Historical", "Action", - "Martial Arts", "Drama", "Tragedy"], - }, - }), - # mutliple values for 'lang' (#4093) - ("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", { - "options": (("lang", "fr,it"),), - "count": 2, - "keyword": { - "manga" : "Souten no Koumori", - "lang" : "re:fr|it", - "language": "re:French|Italian", - }, - }), - ("https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", { - "options": (("lang", "en"),), - "count": ">= 100", - }), - ("https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", { - "count": 1, - }), - ("https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", { - "count": ">= 20", - }) - ) + example = ("https://mangadex.org/title" + "/01234567-89ab-cdef-0123-456789abcdef") def chapters(self): return self.api.manga_feed(self.uuid) @@ -199,7 +142,7 @@ class MangadexFeedExtractor(MangadexExtractor): """Extractor for chapters from your Followed Feed""" subcategory = "feed" pattern = BASE_PATTERN + r"/title/feed$()" - test = ("https://mangadex.org/title/feed",) + example = "https://mangadex.org/title/feed" def chapters(self): return self.api.user_follows_manga_feed() diff --git a/gallery_dl/extractor/mangafox.py b/gallery_dl/extractor/mangafox.py index 8478b8de..d5907532 100644 --- a/gallery_dl/extractor/mangafox.py +++ b/gallery_dl/extractor/mangafox.py @@ -20,23 +20,13 @@ class MangafoxChapterExtractor(ChapterExtractor): root = "https://m.fanfox.net" pattern = BASE_PATTERN + \ r"(/manga/[^/?#]+/((?:v([^/?#]+)/)?c(\d+)([^/?#]*)))" - test = ( - ("http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", { - "keyword": "5661dab258d42d09d98f194f7172fb9851a49766", - "content": "5c50c252dcf12ffecf68801f4db8a2167265f66c", - }), - ("http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/"), - ("http://fanfox.net/manga/black_clover/vTBD/c295/1.html"), - ) + example = "https://fanfox.net/manga/TITLE/v01/c001/1.html" def __init__(self, match): base, self.cstr, self.volume, self.chapter, self.minor = match.groups() self.urlbase = self.root + base ChapterExtractor.__init__(self, match, self.urlbase + "/1.html") - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def metadata(self, page): manga, pos = text.extract(page, "<title>", "") count, pos = text.extract( @@ -73,36 +63,7 @@ class MangafoxMangaExtractor(MangaExtractor): root = "https://m.fanfox.net" chapterclass = MangafoxChapterExtractor pattern = BASE_PATTERN + r"(/manga/[^/?#]+)/?$" - test = ( - ("https://fanfox.net/manga/kanojo_mo_kanojo", { - "pattern": MangafoxChapterExtractor.pattern, - "count": ">=60", - "keyword": { - "author": "HIROYUKI", - "chapter": int, - "chapter_minor": r"re:^(\.\d+)?$", - "chapter_string": r"re:(v\d+/)?c\d+", - "date": "type:datetime", - "description": "High school boy Naoya gets a confession from M" - "omi, a cute and friendly girl. However, Naoya " - "already has a girlfriend, Seki... but Momi is " - "too good a catch to let go. Momi and Nagoya's " - "goal becomes clear: convince Seki to accept be" - "ing an item with the two of them. Will she bud" - "ge?", - "lang": "en", - "language": "English", - "manga": "Kanojo mo Kanojo", - "tags": ["Comedy", "Romance", "School Life", "Shounen"], - "volume": int, - }, - }), - ("https://mangafox.me/manga/shangri_la_frontier", { - "pattern": MangafoxChapterExtractor.pattern, - "count": ">=45", - }), - ("https://m.fanfox.net/manga/sentai_daishikkaku"), - ) + example = "https://fanfox.net/manga/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/mangahere.py b/gallery_dl/extractor/mangahere.py index 745231b1..e8ee8612 100644 --- a/gallery_dl/extractor/mangahere.py +++ b/gallery_dl/extractor/mangahere.py @@ -25,18 +25,7 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor): """Extractor for manga-chapters from mangahere.cc""" pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]/manga/" r"([^/]+(?:/v0*(\d+))?/c([^/?#]+))") - test = ( - ("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", { - "keyword": "7c98d7b50a47e6757b089aa875a53aa970cac66f", - "content": "708d475f06893b88549cbd30df1e3f9428f2c884", - }), - # URLs without HTTP scheme (#1070) - ("https://www.mangahere.cc/manga/beastars/c196/1.html", { - "pattern": "https://zjcdn.mangahere.org/.*", - }), - ("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"), - ("http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/"), - ) + example = "https://www.mangahere.cc/manga/TITLE/c001/1.html" def __init__(self, match): self.part, self.volume, self.chapter = match.groups() @@ -95,24 +84,7 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor): chapterclass = MangahereChapterExtractor pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]" r"(/manga/[^/?#]+/?)(?:#.*)?$") - test = ( - ("https://www.mangahere.cc/manga/aria/", { - "url": "9c2e54ec42e9a87ad53096c328b33c90750af3e4", - "keyword": "71503c682c5d0c277a50409a8c5fd78e871e3d69", - "count": 71, - }), - ("https://www.mangahere.cc/manga/hiyokoi/#50", { - "url": "654850570aa03825cd57e2ae2904af489602c523", - "keyword": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", - }), - # adult filter (#556) - ("http://www.mangahere.cc/manga/gunnm_mars_chronicle/", { - "pattern": MangahereChapterExtractor.pattern, - "count": ">= 50", - }), - ("https://www.mangahere.co/manga/aria/"), - ("https://m.mangahere.co/manga/aria/"), - ) + example = "https://www.mangahere.cc/manga/TITLE" def _init(self): self.cookies.set("isAdult", "1", domain="www.mangahere.cc") diff --git a/gallery_dl/extractor/mangakakalot.py b/gallery_dl/extractor/mangakakalot.py index e397586e..0183b25f 100644 --- a/gallery_dl/extractor/mangakakalot.py +++ b/gallery_dl/extractor/mangakakalot.py @@ -19,30 +19,18 @@ BASE_PATTERN = r"(?:https?://)?(?:ww[\dw]?\.)?mangakakalot\.tv" class MangakakalotBase(): """Base class for mangakakalot extractors""" category = "mangakakalot" - root = "https://ww3.mangakakalot.tv" + root = "https://ww6.mangakakalot.tv" class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): """Extractor for manga chapters from mangakakalot.tv""" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/chapter[_-][^/?#]+)" - test = ( - ("https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", { - "pattern": r"https://cm\.blazefast\.co" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", - "keyword": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", - "count": 9, - }), - ("https://mangakakalot.tv/chapter" - "/hatarakanai_futari_the_jobless_siblings/chapter_20.1"), - ) + example = "https://ww6.mangakakalot.tv/chapter/manga-ID/chapter-01" def __init__(self, match): self.path = match.group(1) ChapterExtractor.__init__(self, match, self.root + self.path) - def _init(self): - self.session.headers['Referer'] = self.root + "/" - def metadata(self, page): _ , pos = text.extract(page, '', '<') manga , pos = text.extract(page, '', '<', pos) @@ -78,13 +66,7 @@ class MangakakalotMangaExtractor(MangakakalotBase, MangaExtractor): """Extractor for manga from mangakakalot.tv""" chapterclass = MangakakalotChapterExtractor pattern = BASE_PATTERN + r"(/manga/[^/?#]+)" - test = ( - ("https://ww3.mangakakalot.tv/manga/manga-jk986845", { - "pattern": MangakakalotChapterExtractor.pattern, - "count": ">= 30", - }), - ("https://mangakakalot.tv/manga/lk921810"), - ) + example = "https://ww6.mangakakalot.tv/manga/manga-ID" def chapters(self, page): data = {"lang": "en", "language": "English"} diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index 807bc5ee..46019ad8 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -23,8 +23,6 @@ class ManganeloBase(): super().__init__(match, "https://" + domain + path) def _init(self): - self.session.headers['Referer'] = self.root + "/" - if self._match_chapter is None: ManganeloBase._match_chapter = re.compile( r"(?:[Vv]ol\.?\s*(\d+)\s?)?" @@ -55,27 +53,7 @@ class ManganeloBase(): class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor): """Extractor for manga chapters from manganelo.com""" pattern = BASE_PATTERN + r"(/(?:manga-\w+|chapter/\w+)/chapter[-_][^/?#]+)" - test = ( - ("https://chapmanganato.com/manga-gn983696/chapter-23", { - "pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23" - r"/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", - "keyword": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", - "count": 25, - }), - ("https://chapmanganelo.com/manga-ti107776/chapter-4", { - "pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92" - r"/08/ti970565/chapter_4_caster/\d+-o\.jpg", - "keyword": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", - "count": 45, - }), - ("https://chapmanganato.com/manga-no991297/chapter-8", { - "keyword": {"chapter": 8, "chapter_minor": "-1"}, - "count": 20, - }), - ("https://readmanganato.com/manga-gn983696/chapter-23"), - ("https://manganelo.com/chapter/gamers/chapter_15"), - ("https://manganelo.com/chapter/gq921227/chapter_23"), - ) + example = "https://chapmanganato.com/manga-ID/chapter-01" def metadata(self, page): extr = text.extract_from(page) @@ -104,19 +82,7 @@ class ManganeloMangaExtractor(ManganeloBase, MangaExtractor): """Extractor for manga from manganelo.com""" chapterclass = ManganeloChapterExtractor pattern = BASE_PATTERN + r"(/(?:manga[-/]|read_)\w+)/?$" - test = ( - ("https://chapmanganato.com/manga-gn983696", { - "pattern": ManganeloChapterExtractor.pattern, - "count": ">= 25", - }), - ("https://m.manganelo.com/manga-ti107776", { - "pattern": ManganeloChapterExtractor.pattern, - "count": ">= 12", - }), - ("https://readmanganato.com/manga-gn983696"), - ("https://manganelo.com/manga/read_otome_no_teikoku"), - ("https://manganelo.com/manga/ol921234/"), - ) + example = "https://manganato.com/manga-ID" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index a0d1e80b..63aaf917 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -35,39 +35,7 @@ class MangaparkBase(): class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor): """Extractor for manga-chapters from mangapark.net""" pattern = BASE_PATTERN + r"/title/[^/?#]+/(\d+)" - test = ( - ("https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", { - "count": 70, - "pattern": r"https://[\w-]+\.mpcdn\.org/comic/2002/e67" - r"/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg" - r"\?acc=[^&#]+&exp=\d+", - "keyword": { - "artist": [], - "author": ["Amano Kozue"], - "chapter": 60, - "chapter_id": 6710214, - "chapter_minor": ".2", - "count": 70, - "date": "dt:2022-01-15 09:25:03", - "extension": "jpeg", - "filename": str, - "genre": ["adventure", "comedy", "drama", "sci_fi", - "shounen", "slice_of_life"], - "lang": "en", - "language": "English", - "manga": "Aria", - "manga_id": 114972, - "page": int, - "source": "Koala", - "title": "Special Navigation - Aquaria Ii", - "volume": 12, - }, - }), - ("https://mangapark.com/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.org/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.io/title/114972-aria/6710214-en-ch.60.2"), - ("https://mangapark.me/title/114972-aria/6710214-en-ch.60.2"), - ) + example = "https://mangapark.net/title/MANGA/12345-en-ch.01" def __init__(self, match): self.root = text.root_from_url(match.group(0)) @@ -115,41 +83,7 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor): """Extractor for manga from mangapark.net""" subcategory = "manga" pattern = BASE_PATTERN + r"/title/(\d+)(?:-[^/?#]*)?/?$" - test = ( - ("https://mangapark.net/title/114972-aria", { - "count": 141, - "pattern": MangaparkChapterExtractor.pattern, - "keyword": { - "chapter": int, - "chapter_id": int, - "chapter_minor": str, - "date": "type:datetime", - "lang": "en", - "language": "English", - "manga_id": 114972, - "source": "re:Horse|Koala", - "source_id": int, - "title": str, - "volume": int, - }, - }), - # 'source' option - ("https://mangapark.net/title/114972-aria", { - "options": (("source", "koala"),), - "count": 70, - "pattern": MangaparkChapterExtractor.pattern, - "keyword": { - "source": "Koala", - "source_id": 15150116, - }, - }), - ("https://mangapark.com/title/114972-"), - ("https://mangapark.com/title/114972"), - ("https://mangapark.com/title/114972-aria"), - ("https://mangapark.org/title/114972-aria"), - ("https://mangapark.io/title/114972-aria"), - ("https://mangapark.me/title/114972-aria"), - ) + example = "https://mangapark.net/title/12345-MANGA" def __init__(self, match): self.root = text.root_from_url(match.group(0)) diff --git a/gallery_dl/extractor/mangaread.py b/gallery_dl/extractor/mangaread.py index 74c239e9..8f193741 100644 --- a/gallery_dl/extractor/mangaread.py +++ b/gallery_dl/extractor/mangaread.py @@ -35,56 +35,7 @@ class MangareadChapterExtractor(MangareadBase, ChapterExtractor): """Extractor for manga-chapters from mangaread.org""" pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org" r"(/manga/[^/?#]+/[^/?#]+)") - test = ( - ("https://www.mangaread.org/manga/one-piece/chapter-1053-3/", { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 11, - "keyword": { - "manga" : "One Piece", - "title" : "", - "chapter" : 1053, - "chapter_minor": ".3", - "tags" : ["Oda Eiichiro"], - "lang" : "en", - "language": "English", - } - }), - ("https://www.mangaread.org/manga/one-piece/chapter-1000000/", { - "exception": exception.NotFoundError, - }), - (("https://www.mangaread.org" - "/manga/kanan-sama-wa-akumade-choroi/chapter-10/"), { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 9, - "keyword": { - "manga" : "Kanan-sama wa Akumade Choroi", - "title" : "", - "chapter" : 10, - "chapter_minor": "", - "tags" : list, - "lang" : "en", - "language": "English", - } - }), - # 'Chapter146.5' - # ^^ no whitespace - ("https://www.mangaread.org/manga/above-all-gods/chapter146-5/", { - "pattern": (r"https://www\.mangaread\.org/wp-content/uploads" - r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"), - "count": 6, - "keyword": { - "manga" : "Above All Gods", - "title" : "", - "chapter" : 146, - "chapter_minor": ".5", - "tags" : list, - "lang" : "en", - "language": "English", - } - }), - ) + example = "https://www.mangaread.org/manga/MANGA/chapter-01/" def metadata(self, page): tags = text.extr(page, 'class="wp-manga-tags-list">', '') @@ -108,50 +59,7 @@ class MangareadMangaExtractor(MangareadBase, MangaExtractor): """Extractor for manga from mangaread.org""" chapterclass = MangareadChapterExtractor pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$" - test = ( - ("https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", { - "pattern": (r"https://www\.mangaread\.org/manga" - r"/kanan-sama-wa-akumade-choroi" - r"/chapter-\d+(-.+)?/"), - "count" : ">= 13", - "keyword": { - "manga" : "Kanan-sama wa Akumade Choroi", - "author" : ["nonco"], - "artist" : ["nonco"], - "type" : "Manga", - "genres" : ["Comedy", "Romance", "Shounen", "Supernatural"], - "rating" : float, - "release": 2022, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : list, - "description": str, - } - }), - ("https://www.mangaread.org/manga/one-piece", { - "pattern": (r"https://www\.mangaread\.org/manga" - r"/one-piece/chapter-\d+(-.+)?/"), - "count" : ">= 1066", - "keyword": { - "manga" : "One Piece", - "author" : ["Oda Eiichiro"], - "artist" : ["Oda Eiichiro"], - "type" : "Manga", - "genres" : list, - "rating" : float, - "release": 1997, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : ["One Piece"], - "description": str, - } - }), - ("https://www.mangaread.org/manga/doesnotexist", { - "exception": exception.HttpError, - }), - ) + example = "https://www.mangaread.org/manga/MANGA" def chapters(self, page): if 'class="error404' in page: diff --git a/gallery_dl/extractor/mangasee.py b/gallery_dl/extractor/mangasee.py index 00c89c1e..72613325 100644 --- a/gallery_dl/extractor/mangasee.py +++ b/gallery_dl/extractor/mangasee.py @@ -36,54 +36,7 @@ class MangaseeBase(): class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor): pattern = (r"(?:https?://)?(mangasee123|manga4life)\.com" r"(/read-online/[^/?#]+\.html)") - test = ( - (("https://mangasee123.com/read-online" - "/Tokyo-Innocent-chapter-4.5-page-1.html"), { - "pattern": r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", - "count": 8, - "keyword": { - "author": ["NARUMI Naru"], - "chapter": 4, - "chapter_minor": ".5", - "chapter_string": "100045", - "count": 8, - "date": "dt:2020-01-20 21:52:53", - "extension": "png", - "filename": r"re:0004\.5-00\d", - "genre": ["Comedy", "Fantasy", "Harem", "Romance", "Shounen", - "Supernatural"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Tokyo Innocent", - "page": int, - "title": "", - }, - }), - (("https://manga4life.com/read-online" - "/One-Piece-chapter-1063-page-1.html"), { - "pattern": r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", - "count": 13, - "keyword": { - "author": ["ODA Eiichiro"], - "chapter": 1063, - "chapter_minor": "", - "chapter_string": "110630", - "count": 13, - "date": "dt:2022-10-16 17:32:54", - "extension": "png", - "filename": r"re:1063-0\d\d", - "genre": ["Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Shounen"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "One Piece", - "page": int, - "title": "", - }, - }), - ) + example = "https://mangasee123.com/read-online/MANGA-chapter-1-page-1.html" def __init__(self, match): if match.group(1) == "manga4life": @@ -134,45 +87,7 @@ class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor): class MangaseeMangaExtractor(MangaseeBase, MangaExtractor): chapterclass = MangaseeChapterExtractor pattern = r"(?:https?://)?(mangasee123|manga4life)\.com(/manga/[^/?#]+)" - test = ( - (("https://mangasee123.com/manga" - "/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai"), { - "pattern": MangaseeChapterExtractor.pattern, - "count": ">= 17", - "keyword": { - "author": ["TAKASE Masaya"], - "chapter": int, - "chapter_minor": r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date": "type:datetime", - "genre": ["Comedy", "Romance", "School Life", "Shounen", - "Slice of Life"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "title": "", - }, - }), - ("https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", { - "pattern": MangaseeChapterExtractor.pattern, - "count": ">= 50", - "keyword": { - "author": ["Canno"], - "chapter": int, - "chapter_minor": r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date": "type:datetime", - "genre": ["Comedy", "Romance", "School Life", "Seinen", - "Shoujo Ai"], - "index": "1", - "lang": "en", - "language": "English", - "manga": "Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "title": "" - }, - }), - ) + example = "https://mangasee123.com/manga/MANGA" def __init__(self, match): if match.group(1) == "manga4life": diff --git a/gallery_dl/extractor/mangoxo.py b/gallery_dl/extractor/mangoxo.py index cca18b13..b208f035 100644 --- a/gallery_dl/extractor/mangoxo.py +++ b/gallery_dl/extractor/mangoxo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -80,25 +80,7 @@ class MangoxoAlbumExtractor(MangoxoExtractor): directory_fmt = ("{category}", "{channel[name]}", "{album[name]}") archive_fmt = "{album[id]}_{num}" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/album/(\w+)" - test = ("https://www.mangoxo.com/album/lzVOv1Q9", { - "url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", - "keyword": { - "channel": { - "id": "gaxO16d8", - "name": "Phoenix", - "cover": str, - }, - "album": { - "id": "lzVOv1Q9", - "name": "re:池永康晟 Ikenaga Yasunari 透出古朴", - "date": "dt:2019-03-22 14:42:00", - "description": str, - }, - "id": int, - "num": int, - "count": 65, - }, - }) + example = "https://www.mangoxo.com/album/ID" def __init__(self, match): MangoxoExtractor.__init__(self, match) @@ -163,11 +145,7 @@ class MangoxoChannelExtractor(MangoxoExtractor): """Extractor for all albums on a mangoxo channel""" subcategory = "channel" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/(\w+)/album" - test = ("https://www.mangoxo.com/phoenix/album", { - "pattern": MangoxoAlbumExtractor.pattern, - "range": "1-30", - "count": "> 20", - }) + example = "https://www.mangoxo.com/USER/album" def __init__(self, match): MangoxoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 3bed955c..3c2b03ee 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -106,29 +106,7 @@ class MastodonUserExtractor(MastodonExtractor): """Extractor for all images of an account/user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)(?:/media)?/?$" - test = ( - ("https://mastodon.social/@jk", { - "pattern": r"https://files.mastodon.social/media_attachments" - r"/files/(\d+/){3,}original/\w+", - "range": "1-60", - "count": 60, - }), - ("https://pawoo.net/@yoru_nine/", { - "range": "1-60", - "count": 60, - }), - ("https://baraag.net/@pumpkinnsfw"), - ("https://mastodon.social/@yoru_nine@pawoo.net", { - "pattern": r"https://mastodon\.social/media_proxy/\d+/original", - "range": "1-10", - "count": 10, - }), - ("https://mastodon.social/@id:10843"), - ("https://mastodon.social/users/id:10843"), - ("https://mastodon.social/users/jk"), - ("https://mastodon.social/users/yoru_nine@pawoo.net"), - ("https://mastodon.social/web/@jk"), - ) + example = "https://mastodon.social/@USER" def statuses(self): api = MastodonAPI(self) @@ -144,11 +122,7 @@ class MastodonBookmarkExtractor(MastodonExtractor): """Extractor for mastodon bookmarks""" subcategory = "bookmark" pattern = BASE_PATTERN + r"/bookmarks" - test = ( - ("https://mastodon.social/bookmarks"), - ("https://pawoo.net/bookmarks"), - ("https://baraag.net/bookmarks"), - ) + example = "https://mastodon.social/bookmarks" def statuses(self): return MastodonAPI(self).account_bookmarks() @@ -157,16 +131,8 @@ class MastodonBookmarkExtractor(MastodonExtractor): class MastodonFollowingExtractor(MastodonExtractor): """Extractor for followed mastodon users""" subcategory = "following" - pattern = BASE_PATTERN + r"/users/([^/?#]+)/following" - test = ( - ("https://mastodon.social/users/0x4f/following", { - "extractor": False, - "count": ">= 20", - }), - ("https://mastodon.social/users/id:10843/following"), - ("https://pawoo.net/users/yoru_nine/following"), - ("https://baraag.net/users/pumpkinnsfw/following"), - ) + pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)/following" + example = "https://mastodon.social/@USER/following" def items(self): api = MastodonAPI(self) @@ -181,21 +147,7 @@ class MastodonStatusExtractor(MastodonExtractor): """Extractor for images from a status""" subcategory = "status" pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)" - test = ( - ("https://mastodon.social/@jk/103794036899778366", { - "count": 4, - "keyword": { - "count": 4, - "num": int, - }, - }), - ("https://pawoo.net/@yoru_nine/105038878897832922", { - "content": "b52e807f8ab548d6f896b09218ece01eba83987a", - }), - ("https://baraag.net/@pumpkinnsfw/104364170556898443", { - "content": "67748c1b828c58ad60d0fe5729b59fb29c872244", - }), - ) + example = "https://mastodon.social/@USER/12345" def statuses(self): return (MastodonAPI(self).status(self.item),) diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 8c717581..95b83b62 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -85,24 +85,7 @@ class MisskeyUserExtractor(MisskeyExtractor): """Extractor for all images of a Misskey user""" subcategory = "user" pattern = BASE_PATTERN + r"/@([^/?#]+)/?$" - test = ( - ("https://misskey.io/@lithla", { - "pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", - "range": "1-50", - "count": 50, - }), - ("https://misskey.io/@blooddj@pawoo.net", { - "range": "1-50", - "count": 50, - }), - ("https://lesbian.energy/@rerorero", { - "pattern": r"https://lesbian.energy/files/\w+", - "range": "1-50", - "count": 50, - }), - ("https://lesbian.energy/@nano@mk.yopo.work"), - ("https://sushi.ski/@ui@misskey.04.si"), - ) + example = "https://misskey.io/@USER" def notes(self): return self.api.users_notes(self.api.user_id_by_username(self.item)) @@ -112,13 +95,7 @@ class MisskeyFollowingExtractor(MisskeyExtractor): """Extractor for followed Misskey users""" subcategory = "following" pattern = BASE_PATTERN + r"/@([^/?#]+)/following" - test = ( - ("https://misskey.io/@blooddj@pawoo.net/following", { - "extractor": False, - "count": ">= 6", - }), - ("https://sushi.ski/@hatusimo_sigure/following"), - ) + example = "https://misskey.io/@USER/following" def items(self): user_id = self.api.user_id_by_username(self.item) @@ -136,21 +113,7 @@ class MisskeyNoteExtractor(MisskeyExtractor): """Extractor for images from a Note""" subcategory = "note" pattern = BASE_PATTERN + r"/notes/(\w+)" - test = ( - ("https://misskey.io/notes/9bhqfo835v", { - "pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", - "count": 4, - }), - ("https://misskey.io/notes/9brq7z1re6"), - ("https://sushi.ski/notes/9bm3x4ksqw", { - "pattern": r"https://media\.sushi\.ski/files/[\w-]+\.png", - "count": 1, - }), - ("https://lesbian.energy/notes/995ig09wqy", { - "count": 1, - }), - ("https://lesbian.energy/notes/96ynd9w5kc"), - ) + example = "https://misskey.io/notes/98765" def notes(self): return (self.api.notes_show(self.item),) @@ -160,12 +123,7 @@ class MisskeyFavoriteExtractor(MisskeyExtractor): """Extractor for favorited notes""" subcategory = "favorite" pattern = BASE_PATTERN + r"/(?:my|api/i)/favorites" - test = ( - ("https://misskey.io/my/favorites"), - ("https://misskey.io/api/i/favorites"), - ("https://lesbian.energy/my/favorites"), - ("https://sushi.ski/my/favorites"), - ) + example = "https://misskey.io/my/favorites" def notes(self): return self.api.i_favorites() diff --git a/gallery_dl/extractor/moebooru.py b/gallery_dl/extractor/moebooru.py index 1e56bde9..145dd51d 100644 --- a/gallery_dl/extractor/moebooru.py +++ b/gallery_dl/extractor/moebooru.py @@ -92,94 +92,12 @@ BASE_PATTERN = MoebooruExtractor.update({ }) -class MoebooruPostExtractor(MoebooruExtractor): - subcategory = "post" - archive_fmt = "{id}" - pattern = BASE_PATTERN + r"/post/show/(\d+)" - test = ( - ("https://yande.re/post/show/51824", { - "content": "59201811c728096b2d95ce6896fd0009235fe683", - "options": (("tags", True),), - "keyword": { - "tags_artist": "sasaki_tamaru", - "tags_circle": "softhouse_chara", - "tags_copyright": "ouzoku", - "tags_general": str, - }, - }), - ("https://konachan.com/post/show/205189", { - "content": "674e75a753df82f5ad80803f575818b8e46e4b65", - "options": (("tags", True),), - "keyword": { - "tags_artist": "patata", - "tags_character": "clownpiece", - "tags_copyright": "touhou", - "tags_general": str, - }, - }), - ("https://yande.re/post/show/993156", { - "content": "fed722bd90f48de41ec163692befc701056e2b1e", - "options": (("notes", True),), - "keyword": { - "notes": [ - { - "id": 7096, - "x" : 90, - "y" : 626, - "width" : 283, - "height": 529, - "body" : "Please keep this as a secret for me!!", - }, - { - "id": 7095, - "x" : 900, - "y" : 438, - "width" : 314, - "height": 588, - "body" : "The facts that I love playing games", - }, - ], - }, - }), - ("https://lolibooru.moe/post/show/281305/", { - "content": "a331430223ffc5b23c31649102e7d49f52489b57", - "options": (("notes", True),), - "keyword": { - "notes": list, - }, - }), - ("https://konachan.net/post/show/205189"), - ("https://www.sakugabooru.com/post/show/125570"), - ("https://lolibooru.moe/post/show/287835"), - ) - - def __init__(self, match): - MoebooruExtractor.__init__(self, match) - self.post_id = match.group(match.lastindex) - - def posts(self): - params = {"tags": "id:" + self.post_id} - return self.request(self.root + "/post.json", params=params).json() - - class MoebooruTagExtractor(MoebooruExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/post\?(?:[^&#]*&)*tags=([^&#]*)" - test = ( - ("https://yande.re/post?tags=ouzoku+armor", { - "content": "59201811c728096b2d95ce6896fd0009235fe683", - }), - ("https://konachan.com/post?tags=patata", { - "content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", - }), - # empty 'tags' (#4354) - ("https://konachan.com/post?tags="), - ("https://konachan.net/post?tags=patata"), - ("https://www.sakugabooru.com/post?tags=nichijou"), - ("https://lolibooru.moe/post?tags=ruu_%28tksymkw%29"), - ) + example = "https://yande.re/post?tags=TAG" def __init__(self, match): MoebooruExtractor.__init__(self, match) @@ -199,17 +117,7 @@ class MoebooruPoolExtractor(MoebooruExtractor): directory_fmt = ("{category}", "pool", "{pool}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/pool/show/(\d+)" - test = ( - ("https://yande.re/pool/show/318", { - "content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", - }), - ("https://konachan.com/pool/show/95", { - "content": "cf0546e38a93c2c510a478f8744e60687b7a8426", - }), - ("https://konachan.net/pool/show/95"), - ("https://www.sakugabooru.com/pool/show/54"), - ("https://lolibooru.moe/pool/show/239"), - ) + example = "https://yande.re/pool/show/12345" def __init__(self, match): MoebooruExtractor.__init__(self, match) @@ -223,25 +131,28 @@ class MoebooruPoolExtractor(MoebooruExtractor): return self._pagination(self.root + "/post.json", params) +class MoebooruPostExtractor(MoebooruExtractor): + subcategory = "post" + archive_fmt = "{id}" + pattern = BASE_PATTERN + r"/post/show/(\d+)" + example = "https://yande.re/post/show/12345" + + def __init__(self, match): + MoebooruExtractor.__init__(self, match) + self.post_id = match.group(match.lastindex) + + def posts(self): + params = {"tags": "id:" + self.post_id} + return self.request(self.root + "/post.json", params=params).json() + + class MoebooruPopularExtractor(MoebooruExtractor): subcategory = "popular" directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" pattern = BASE_PATTERN + \ r"/post/popular_(by_(?:day|week|month)|recent)(?:\?([^#]*))?" - test = ( - ("https://yande.re/post/popular_by_month?month=6&year=2014", { - "count": 40, - }), - ("https://yande.re/post/popular_recent"), - ("https://konachan.com/post/popular_by_month?month=11&year=2010", { - "count": 20, - }), - ("https://konachan.com/post/popular_recent"), - ("https://konachan.net/post/popular_recent"), - ("https://www.sakugabooru.com/post/popular_recent"), - ("https://lolibooru.moe/post/popular_recent"), - ) + example = "https://yande.re/post/popular_by_month?year=YYYY&month=MM" def __init__(self, match): MoebooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/myhentaigallery.py b/gallery_dl/extractor/myhentaigallery.py index 3301da97..33a22849 100644 --- a/gallery_dl/extractor/myhentaigallery.py +++ b/gallery_dl/extractor/myhentaigallery.py @@ -13,26 +13,11 @@ from .. import text, exception class MyhentaigalleryGalleryExtractor(GalleryExtractor): """Extractor for image galleries from myhentaigallery.com""" category = "myhentaigallery" + root = "https://myhentaigallery.com" directory_fmt = ("{category}", "{gallery_id} {artist:?[/] /J, }{title}") pattern = (r"(?:https?://)?myhentaigallery\.com" r"/gallery/(?:thumbnails|show)/(\d+)") - test = ( - ("https://myhentaigallery.com/gallery/thumbnails/16247", { - "pattern": r"https://images.myhentaicomics\.com/imagesgallery" - r"/images/[^/]+/original/\d+\.jpg", - "keyword": { - "artist" : list, - "count" : 11, - "gallery_id": 16247, - "group" : list, - "parodies" : list, - "tags" : ["Giantess"], - "title" : "Attack Of The 50ft Woman 1", - }, - }), - ("https://myhentaigallery.com/gallery/show/16247/1"), - ) - root = "https://myhentaigallery.com" + example = "https://myhentaigallery.com/gallery/thumbnails/12345" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/myportfolio.py b/gallery_dl/extractor/myportfolio.py index fd16f247..7620d088 100644 --- a/gallery_dl/extractor/myportfolio.py +++ b/gallery_dl/extractor/myportfolio.py @@ -22,27 +22,7 @@ class MyportfolioGalleryExtractor(Extractor): pattern = (r"(?:myportfolio:(?:https?://)?([^/]+)|" r"(?:https?://)?([\w-]+\.myportfolio\.com))" r"(/[^/?#]+)?") - test = ( - ("https://andrewling.myportfolio.com/volvo-xc-90-hybrid", { - "url": "acea0690c76db0e5cf267648cefd86e921bc3499", - "keyword": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", - }), - ("https://andrewling.myportfolio.com/", { - "pattern": r"https://andrewling\.myportfolio\.com/[^/?#+]+$", - "count": ">= 6", - }), - ("https://stevenilousphotography.myportfolio.com/society", { - "exception": exception.NotFoundError, - }), - # custom domain - ("myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", { - "count": 3, - }), - ("myportfolio:https://tooco.com.ar/", { - "pattern": pattern, - "count": ">= 40", - }), - ) + example = "https://USER.myportfolio.com/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/naver.py b/gallery_dl/extractor/naver.py index 7e94b1cc..55faf9e7 100644 --- a/gallery_dl/extractor/naver.py +++ b/gallery_dl/extractor/naver.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -27,17 +27,7 @@ class NaverPostExtractor(NaverBase, GalleryExtractor): archive_fmt = "{blog[id]}_{post[num]}_{num}" pattern = (r"(?:https?://)?blog\.naver\.com/" r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)") - test = ( - ("https://blog.naver.com/rlfqjxm0/221430673006", { - "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", - }), - (("https://blog.naver.com/PostView.nhn" - "?blogId=rlfqjxm0&logNo=221430673006"), { - "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", - }), - ) + example = "https://blog.naver.com/BLOGID/12345" def __init__(self, match): blog_id = match.group(1) @@ -84,18 +74,7 @@ class NaverBlogExtractor(NaverBase, Extractor): categorytransfer = True pattern = (r"(?:https?://)?blog\.naver\.com/" r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)") - test = ( - ("https://blog.naver.com/gukjung", { - "pattern": NaverPostExtractor.pattern, - "count": 12, - "range": "1-12", - }), - ("https://blog.naver.com/PostList.nhn?blogId=gukjung", { - "pattern": NaverPostExtractor.pattern, - "count": 12, - "range": "1-12", - }), - ) + example = "https://blog.naver.com/BLOGID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/naverwebtoon.py b/gallery_dl/extractor/naverwebtoon.py index cafe4f7f..72ee5b06 100644 --- a/gallery_dl/extractor/naverwebtoon.py +++ b/gallery_dl/extractor/naverwebtoon.py @@ -28,54 +28,7 @@ class NaverwebtoonEpisodeExtractor(NaverwebtoonBase, GalleryExtractor): filename_fmt = "{episode:>03}-{num:>02}.{extension}" archive_fmt = "{title_id}_{episode}_{num}" pattern = BASE_PATTERN + r"/detail(?:\.nhn)?\?([^#]+)" - test = ( - (("https://comic.naver.com/webtoon/detail" - "?titleId=26458&no=1&weekday=tue"), { - "url": "47a956ba8c7a837213d5985f50c569fcff986f75", - "content": "3806b6e8befbb1920048de9888dfce6220f69a60", - "count": 14, - "keyword": { - "author": ["김규삼"], - "artist": ["김규삼"], - "comic": "N의등대-눈의등대", - "count": 14, - "episode": "1", - "extension": "jpg", - "num": int, - "tags": ["스릴러", "완결무료", "완결스릴러"], - "title": "n의 등대 - 눈의 등대 1화", - "title_id": "26458", - }, - }), - (("https://comic.naver.com/challenge/detail" - "?titleId=765124&no=1"), { - "pattern": r"https://image-comic\.pstatic\.net" - r"/user_contents_data/challenge_comic/2021/01/19" - r"/342586/upload_7149856273586337846\.jpeg", - "count": 1, - "keyword": { - "author": ["kemi****"], - "artist": [], - "comic": "우니 모두의 이야기", - "count": 1, - "episode": "1", - "extension": "jpeg", - "filename": "upload_7149856273586337846", - "num": 1, - "tags": ["일상툰", "우니모두의이야기", "퇴사", "입사", "신입사원", - "사회초년생", "회사원", "20대"], - "title": "퇴사하다", - "title_id": "765124", - }, - }), - (("https://comic.naver.com/bestChallenge/detail.nhn" - "?titleId=771467&no=3"), { - "pattern": r"https://image-comic\.pstatic\.net" - r"/user_contents_data/challenge_comic/2021/04/28" - r"/345534/upload_3617293622396203109\.jpeg", - "count": 1, - }), - ) + example = "https://comic.naver.com/webtoon/detail?titleId=12345&no=1" def __init__(self, match): path, query = match.groups() @@ -115,20 +68,7 @@ class NaverwebtoonComicExtractor(NaverwebtoonBase, Extractor): subcategory = "comic" categorytransfer = True pattern = BASE_PATTERN + r"/list(?:\.nhn)?\?([^#]+)" - test = ( - ("https://comic.naver.com/webtoon/list?titleId=22073", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": 32, - }), - ("https://comic.naver.com/challenge/list?titleId=765124", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": 25, - }), - ("https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", { - "pattern": NaverwebtoonEpisodeExtractor.pattern, - "count": ">= 12", - }), - ) + example = "https://comic.naver.com/webtoon/list?titleId=12345" def __init__(self, match): Extractor.__init__(self, match) @@ -145,7 +85,6 @@ class NaverwebtoonComicExtractor(NaverwebtoonBase, Extractor): url = self.root + "/api/article/list" headers = { "Accept": "application/json, text/plain, */*", - "Referer": self.root + "/", } params = { "titleId": self.title_id, diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 8a255280..1bcc915d 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -207,7 +207,6 @@ class NewgroundsExtractor(Extractor): headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", - "Referer": self.root, } sources = self.request(url, headers=headers).json()["sources"] @@ -300,41 +299,7 @@ class NewgroundsImageExtractor(NewgroundsExtractor): pattern = (r"(?:https?://)?(?:" r"(?:www\.)?newgrounds\.com/art/view/([^/?#]+)/[^/?#]+" r"|art\.ngfiles\.com/images/\d+/\d+_([^_]+)_([^.]+))") - test = ( - ("https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", { - "url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", - "content": "8f395e08333eb2457ba8d8b715238f8910221365", - "keyword": { - "artist" : ["tomfulp"], - "comment" : "re:Consider this the bottom threshold for ", - "date" : "dt:2009-06-04 14:44:05", - "description": "re:Consider this the bottom threshold for ", - "favorites" : int, - "filename" : "94_tomfulp_ryu-is-hawt", - "height" : 476, - "index" : 94, - "rating" : "e", - "score" : float, - "tags" : ["ryu", "streetfighter"], - "title" : "Ryu is Hawt", - "type" : "article", - "user" : "tomfulp", - "width" : 447, - }, - }), - ("https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", { - "url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", - }), - ("https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", { - "url": "84eec95e663041a80630df72719f231e157e5f5d", - "count": 2, - }), - # "adult" rated (#2456) - ("https://www.newgrounds.com/art/view/kekiiro/red", { - "options": (("username", None),), - "count": 1, - }), - ) + example = "https://www.newgrounds.com/art/view/USER/TITLE" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -354,104 +319,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor): subcategory = "media" pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" r"(/(?:portal/view|audio/listen)/\d+)") - test = ( - ("https://www.newgrounds.com/portal/view/595355", { - "pattern": r"https://uploads\.ungrounded\.net/alternate/564000" - r"/564957_alternate_31\.mp4\?1359712249", - "keyword": { - "artist" : ["kickinthehead", "danpaladin", "tomfulp"], - "comment" : "re:My fan trailer for Alien Hominid HD!", - "date" : "dt:2013-02-01 09:50:49", - "description": "Fan trailer for Alien Hominid HD!", - "favorites" : int, - "filename" : "564957_alternate_31", - "index" : 595355, - "rating" : "e", - "score" : float, - "tags" : ["alienhominid", "trailer"], - "title" : "Alien Hominid Fan Trailer", - "type" : "movie", - "user" : "kickinthehead", - }, - }), - ("https://www.newgrounds.com/audio/listen/609768", { - "url": "f4c5490ae559a3b05e46821bb7ee834f93a43c95", - "keyword": { - "artist" : ["zj", "tomfulp"], - "comment" : "re:RECORDED 12-09-2014\n\nFrom The ZJ \"Late ", - "date" : "dt:2015-02-23 19:31:59", - "description": "From The ZJ Report Show!", - "favorites" : int, - "index" : 609768, - "rating" : "", - "score" : float, - "tags" : ["fulp", "interview", "tom", "zj"], - "title" : "ZJ Interviews Tom Fulp!", - "type" : "music.song", - "user" : "zj", - }, - }), - # flash animation (#1257) - ("https://www.newgrounds.com/portal/view/161181/format/flash", { - "pattern": r"https://uploads\.ungrounded\.net/161000" - r"/161181_ddautta_mask__550x281_\.swf\?f1081628129", - "keyword": {"type": "movie"}, - }), - # format selection (#1729) - ("https://www.newgrounds.com/portal/view/758545", { - "options": (("format", "720p"),), - "pattern": r"https://uploads\.ungrounded\.net/alternate/1482000" - r"/1482860_alternate_102516\.720p\.mp4\?\d+", - }), - # "adult" rated (#2456) - ("https://www.newgrounds.com/portal/view/717744", { - "options": (("username", None),), - "count": 1, - }), - # flash game - ("https://www.newgrounds.com/portal/view/829032", { - "pattern": r"https://uploads\.ungrounded\.net/829000" - r"/829032_picovsbeardx\.swf\?f1641968445", - "range": "1", - "keyword": { - "artist" : [ - "dungeonation", - "carpetbakery", - "animalspeakandrews", - "bill", - "chipollo", - "dylz49", - "gappyshamp", - "pinktophat", - "rad", - "shapeshiftingblob", - "tomfulp", - "voicesbycorey", - "psychogoldfish", - ], - "comment" : "re:The children are expendable. Take out the ", - "date" : "dt:2022-01-10 23:00:57", - "description": "Bloodshed in The Big House that Blew...again!", - "favorites" : int, - "index" : 829032, - "post_url" : "https://www.newgrounds.com/portal/view/829032", - "rating" : "m", - "score" : float, - "tags" : [ - "assassin", - "boyfriend", - "darnell", - "nene", - "pico", - "picos-school", - ], - "title" : "PICO VS BEAR DX", - "type" : "game", - "url" : "https://uploads.ungrounded.net/829000" - "/829032_picovsbeardx.swf?f1641968445", - }, - }), - ) + example = "https://www.newgrounds.com/portal/view/12345" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -466,58 +334,35 @@ class NewgroundsArtExtractor(NewgroundsExtractor): """Extractor for all images of a newgrounds user""" subcategory = _path = "art" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/art/?$" - test = ("https://tomfulp.newgrounds.com/art", { - "pattern": NewgroundsImageExtractor.pattern, - "count": ">= 3", - }) + example = "https://USER.newgrounds.com/art" class NewgroundsAudioExtractor(NewgroundsExtractor): """Extractor for all audio submissions of a newgrounds user""" subcategory = _path = "audio" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/audio/?$" - test = ("https://tomfulp.newgrounds.com/audio", { - "pattern": r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3", - "count": ">= 4", - }) + example = "https://USER.newgrounds.com/audio" class NewgroundsMoviesExtractor(NewgroundsExtractor): """Extractor for all movies of a newgrounds user""" subcategory = _path = "movies" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$" - test = ("https://tomfulp.newgrounds.com/movies", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "range": "1-10", - "count": 10, - }) + example = "https://USER.newgrounds.com/movies" class NewgroundsGamesExtractor(NewgroundsExtractor): """Extractor for a newgrounds user's games""" subcategory = _path = "games" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/games/?$" - test = ("https://tomfulp.newgrounds.com/games", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "range": "1-10", - "count": 10, - }) + example = "https://USER.newgrounds.com/games" class NewgroundsUserExtractor(NewgroundsExtractor): """Extractor for a newgrounds user profile""" subcategory = "user" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/?$" - test = ( - ("https://tomfulp.newgrounds.com", { - "pattern": "https://tomfulp.newgrounds.com/art$", - }), - ("https://tomfulp.newgrounds.com", { - "options": (("include", "all"),), - "pattern": "https://tomfulp.newgrounds.com/(art|audio|movies)$", - "count": 3, - }), - ) + example = "https://USER.newgrounds.com" def initialize(self): pass @@ -538,15 +383,7 @@ class NewgroundsFavoriteExtractor(NewgroundsExtractor): directory_fmt = ("{category}", "{user}", "Favorites") pattern = (r"(?:https?://)?([\w-]+)\.newgrounds\.com" r"/favorites(?!/following)(?:/(art|audio|movies))?/?") - test = ( - ("https://tomfulp.newgrounds.com/favorites/art", { - "range": "1-10", - "count": ">= 10", - }), - ("https://tomfulp.newgrounds.com/favorites/audio"), - ("https://tomfulp.newgrounds.com/favorites/movies"), - ("https://tomfulp.newgrounds.com/favorites/"), - ) + example = "https://USER.newgrounds.com/favorites" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -595,11 +432,7 @@ class NewgroundsFollowingExtractor(NewgroundsFavoriteExtractor): """Extractor for a newgrounds user's favorited users""" subcategory = "following" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/favorites/(following)" - test = ("https://tomfulp.newgrounds.com/favorites/following", { - "pattern": NewgroundsUserExtractor.pattern, - "range": "76-125", - "count": 50, - }) + example = "https://USER.newgrounds.com/favorites/following" def items(self): data = {"_extractor": NewgroundsUserExtractor} @@ -620,21 +453,7 @@ class NewgroundsSearchExtractor(NewgroundsExtractor): directory_fmt = ("{category}", "search", "{search_tags}") pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" r"/search/conduct/([^/?#]+)/?\?([^#]+)") - test = ( - ("https://www.newgrounds.com/search/conduct/art?terms=tree", { - "pattern": NewgroundsImageExtractor.pattern, - "keyword": {"search_tags": "tree"}, - "range": "1-10", - "count": 10, - }), - ("https://www.newgrounds.com/search/conduct/movies?terms=tree", { - "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", - "range": "1-10", - "count": 10, - }), - ("https://www.newgrounds.com/search/conduct/audio?advanced=1" - "&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm"), - ) + example = "https://www.newgrounds.com/search/conduct/art?terms=QUERY" def __init__(self, match): NewgroundsExtractor.__init__(self, match) @@ -658,7 +477,6 @@ class NewgroundsSearchExtractor(NewgroundsExtractor): headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", - "Referer": self.root, } params["inner"] = "1" params["page"] = 1 diff --git a/gallery_dl/extractor/nhentai.py b/gallery_dl/extractor/nhentai.py index 4270c84a..09b2b167 100644 --- a/gallery_dl/extractor/nhentai.py +++ b/gallery_dl/extractor/nhentai.py @@ -18,29 +18,7 @@ class NhentaiGalleryExtractor(GalleryExtractor): category = "nhentai" root = "https://nhentai.net" pattern = r"(?:https?://)?nhentai\.net/g/(\d+)" - test = ("https://nhentai.net/g/147850/", { - "url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", - "keyword": { - "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", - "title_en" : str, - "title_ja" : str, - "gallery_id": 147850, - "media_id" : 867789, - "count" : 16, - "date" : 1446050915, - "scanlator" : "", - "artist" : ["morris"], - "group" : list, - "parody" : list, - "characters": list, - "tags" : list, - "type" : "manga", - "lang" : "en", - "language" : "English", - "width" : int, - "height" : int, - }, - }) + example = "https://nhentai.net/g/12345/" def __init__(self, match): url = self.root + "/api/gallery/" + match.group(1) @@ -127,34 +105,18 @@ class NhentaiTagExtractor(NhentaiExtractor): pattern = (r"(?:https?://)?nhentai\.net(" r"/(?:artist|category|character|group|language|parody|tag)" r"/[^/?#]+(?:/popular[^/?#]*)?/?)(?:\?([^#]+))?") - test = ( - ("https://nhentai.net/tag/sole-female/", { - "pattern": NhentaiGalleryExtractor.pattern, - "count": 30, - "range": "1-30", - }), - ("https://nhentai.net/artist/itou-life/"), - ("https://nhentai.net/group/itou-life/"), - ("https://nhentai.net/parody/touhou-project/"), - ("https://nhentai.net/character/patchouli-knowledge/popular"), - ("https://nhentai.net/category/doujinshi/popular-today"), - ("https://nhentai.net/language/english/popular-week"), - ) + example = "https://nhentai.net/tag/TAG/" class NhentaiSearchExtractor(NhentaiExtractor): """Extractor for nhentai search results""" subcategory = "search" pattern = r"(?:https?://)?nhentai\.net(/search/?)\?([^#]+)" - test = ("https://nhentai.net/search/?q=touhou", { - "pattern": NhentaiGalleryExtractor.pattern, - "count": 30, - "range": "1-30", - }) + example = "https://nhentai.net/search/?q=QUERY" class NhentaiFavoriteExtractor(NhentaiExtractor): """Extractor for nhentai favorites""" subcategory = "favorite" pattern = r"(?:https?://)?nhentai\.net(/favorites/?)(?:\?([^#]+))?" - test = ("https://nhentai.net/favorites/",) + example = "https://nhentai.net/favorites/" diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index 66040d8a..b902404c 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,7 +30,6 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor): BaseExtractor.initialize(self) - self.session.headers["Referer"] = self.root + "/" self.user_name = None if self.category == "horne": self._extract_data = self._extract_data_horne @@ -176,10 +175,7 @@ class NijieUserExtractor(NijieExtractor): subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/members\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members.php?id=44"), - ("https://horne.red/members.php?id=58000"), - ) + example = "https://nijie.info/members.php?id=12345" def initialize(self): pass @@ -198,48 +194,7 @@ class NijieIllustrationExtractor(NijieExtractor): """Extractor for all illustrations of a nijie-user""" subcategory = "illustration" pattern = BASE_PATTERN + r"/members_illust\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members_illust.php?id=44", { - "url": "1553e5144df50a676f5947d02469299b401ad6c0", - "keyword": { - "artist_id": 44, - "artist_name": "ED", - "date": "type:datetime", - "description": str, - "extension": "jpg", - "filename": str, - "image_id": int, - "num": int, - "tags": list, - "title": str, - "url": r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", - "user_id": 44, - "user_name": "ED", - }, - }), - ("https://horne.red/members_illust.php?id=58000", { - "pattern": r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+" - r"/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", - "range": "1-20", - "count": 20, - "keyword": { - "artist_id": 58000, - "artist_name": "のえるわ", - "date": "type:datetime", - "description": str, - "image_id": int, - "num": int, - "tags": list, - "title": str, - "url": str, - "user_id": 58000, - "user_name": "のえるわ", - }, - }), - ("https://nijie.info/members_illust.php?id=43", { - "exception": exception.NotFoundError, - }), - ) + example = "https://nijie.info/members_illust.php?id=12345" def image_ids(self): return self._pagination("members_illust") @@ -249,16 +204,7 @@ class NijieDoujinExtractor(NijieExtractor): """Extractor for doujin entries of a nijie user""" subcategory = "doujin" pattern = BASE_PATTERN + r"/members_dojin\.php\?id=(\d+)" - test = ( - ("https://nijie.info/members_dojin.php?id=6782", { - "count": ">= 18", - "keyword": { - "user_id" : 6782, - "user_name": "ジョニー＠アビオン村", - }, - }), - ("https://horne.red/members_dojin.php?id=58000"), - ) + example = "https://nijie.info/members_dojin.php?id=12345" def image_ids(self): return self._pagination("members_dojin") @@ -270,23 +216,7 @@ class NijieFavoriteExtractor(NijieExtractor): directory_fmt = ("{category}", "bookmarks", "{user_id}") archive_fmt = "f_{user_id}_{image_id}_{num}" pattern = BASE_PATTERN + r"/user_like_illust_view\.php\?id=(\d+)" - test = ( - ("https://nijie.info/user_like_illust_view.php?id=44", { - "count": ">= 16", - "keyword": { - "user_id" : 44, - "user_name": "ED", - }, - }), - ("https://horne.red/user_like_illust_view.php?id=58000", { - "range": "1-5", - "count": 5, - "keyword": { - "user_id" : 58000, - "user_name": "のえるわ", - }, - }), - ) + example = "https://nijie.info/user_like_illust_view.php?id=12345" def image_ids(self): return self._pagination("user_like_illust_view") @@ -304,17 +234,7 @@ class NijieNuitaExtractor(NijieExtractor): directory_fmt = ("{category}", "nuita", "{user_id}") archive_fmt = "n_{user_id}_{image_id}_{num}" pattern = BASE_PATTERN + r"/history_nuita\.php\?id=(\d+)" - test = ( - ("https://nijie.info/history_nuita.php?id=728995", { - "range": "1-10", - "count": 10, - "keyword": { - "user_id" : 728995, - "user_name": "莚", - }, - }), - ("https://horne.red/history_nuita.php?id=58000"), - ) + example = "https://nijie.info/history_nuita.php?id=12345" def image_ids(self): return self._pagination("history_nuita") @@ -334,13 +254,7 @@ class NijieFeedExtractor(NijieExtractor): """Extractor for nijie liked user feed""" subcategory = "feed" pattern = BASE_PATTERN + r"/like_user_view\.php" - test = ( - ("https://nijie.info/like_user_view.php", { - "range": "1-10", - "count": 10, - }), - ("https://horne.red/like_user_view.php"), - ) + example = "https://nijie.info/like_user_view.php" def image_ids(self): return self._pagination("like_user_view") @@ -350,14 +264,11 @@ class NijieFeedExtractor(NijieExtractor): return "" -class NijiefollowedExtractor(NijieExtractor): +class NijieFollowedExtractor(NijieExtractor): """Extractor for followed nijie users""" subcategory = "followed" pattern = BASE_PATTERN + r"/like_my\.php" - test = ( - ("https://nijie.info/like_my.php"), - ("https://horne.red/like_my.php"), - ) + example = "https://nijie.info/like_my.php" def items(self): self.login() @@ -383,32 +294,7 @@ class NijieImageExtractor(NijieExtractor): """Extractor for a nijie work/image""" subcategory = "image" pattern = BASE_PATTERN + r"/view(?:_popup)?\.php\?id=(\d+)" - test = ( - ("https://nijie.info/view.php?id=70720", { - "url": "3d654e890212ba823c9647754767336aebc0a743", - "keyword": "41da5d0e178b04f01fe72460185df52fadc3c91b", - "content": "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", - }), - ("https://nijie.info/view.php?id=70724", { - "count": 0, - }), - ("https://nijie.info/view_popup.php?id=70720"), - ("https://horne.red/view.php?id=8716", { - "count": 4, - "keyword": { - "artist_id": 58000, - "artist_name": "のえるわ", - "date": "dt:2018-02-04 14:47:24", - "description": "ノエル「そんなことしなくても、" - "言ってくれたら咥えるのに・・・♡」", - "image_id": 8716, - "tags": ["男の娘", "フェラ", "オリキャラ", "うちのこ"], - "title": "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", - "user_id": 58000, - "user_name": "のえるわ", - }, - }), - ) + example = "https://nijie.info/view.php?id=12345" def __init__(self, match): NijieExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/nitter.py b/gallery_dl/extractor/nitter.py index fda169d8..9f5cc9da 100644 --- a/gallery_dl/extractor/nitter.py +++ b/gallery_dl/extractor/nitter.py @@ -261,58 +261,7 @@ USER_PATTERN = BASE_PATTERN + r"/(i(?:/user/|d:)(\d+)|[^/?#]+)" class NitterTweetsExtractor(NitterExtractor): subcategory = "tweets" pattern = USER_PATTERN + r"(?:/tweets)?(?:$|\?|#)" - test = ( - ("https://nitter.net/supernaturepics", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - "count": 20, - "keyword": { - "author": { - "name": "supernaturepics", - "nick": "Nature Pictures" - }, - "comments": int, - "content": str, - "count": 1, - "date": "type:datetime", - "likes": int, - "quotes": int, - "retweets": int, - "tweet_id": r"re:\d+", - "user": { - "date": "dt:2015-01-12 10:25:00", - "description": "The very best nature pictures.", - "favourites_count": int, - "followers_count": int, - "friends_count": int, - "id": "2976459548", - "name": "supernaturepics", - "nick": "Nature Pictures", - "profile_banner": "https://nitter.net/pic/https%3A%2F%2Fpb" - "s.twimg.com%2Fprofile_banners%2F2976459" - "548%2F1421058583%2F1500x500", - "profile_image": "https://nitter.net/pic/pbs.twimg.com%2Fp" - "rofile_images%2F554585280938659841%2FFLV" - "AlX18.jpeg", - "statuses_count": 1568, - "verified": False, - }, - }, - }), - ("https://nitter.lacontrevoie.fr/supernaturepics", { - "url": "54f4b55f2099dcc248f3fb7bfacf1349e08d8e2d", - "pattern": r"https://nitter\.lacontrevoie\.fr/pic/orig" - r"/media%2FCGMNYZvW0AIVoom\.jpg", - "range": "1", - }), - ("https://nitter.1d4.us/supernaturepics", { - "range": "1", - "keyword": {"user": {"id": "2976459548"}}, - }), - ("https://nitter.kavin.rocks/id:2976459548"), - ("https://nitter.unixfox.eu/supernaturepics"), - ) + example = "https://nitter.net/USER" def tweets(self): return self._pagination("") @@ -321,17 +270,7 @@ class NitterTweetsExtractor(NitterExtractor): class NitterRepliesExtractor(NitterExtractor): subcategory = "replies" pattern = USER_PATTERN + r"/with_replies" - test = ( - ("https://nitter.net/supernaturepics/with_replies", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/with_replies"), - ("https://nitter.1d4.us/supernaturepics/with_replies"), - ("https://nitter.kavin.rocks/id:2976459548/with_replies"), - ("https://nitter.unixfox.eu/i/user/2976459548/with_replies"), - ) + example = "https://nitter.net/USER/with_replies" def tweets(self): return self._pagination("/with_replies") @@ -340,21 +279,7 @@ class NitterRepliesExtractor(NitterExtractor): class NitterMediaExtractor(NitterExtractor): subcategory = "media" pattern = USER_PATTERN + r"/media" - test = ( - ("https://nitter.net/supernaturepics/media", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.kavin.rocks/id:2976459548/media", { - "pattern": r"https://nitter\.kavin\.rocks/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/media"), - ("https://nitter.1d4.us/supernaturepics/media"), - ("https://nitter.unixfox.eu/i/user/2976459548/media"), - ) + example = "https://nitter.net/USER/media" def tweets(self): return self._pagination("/media") @@ -363,17 +288,7 @@ class NitterMediaExtractor(NitterExtractor): class NitterSearchExtractor(NitterExtractor): subcategory = "search" pattern = USER_PATTERN + r"/search" - test = ( - ("https://nitter.net/supernaturepics/search", { - "pattern": r"https://nitter\.net/pic/orig" - r"/media%2F[\w-]+\.(jpg|png)$", - "range": "1-20", - }), - ("https://nitter.lacontrevoie.fr/supernaturepics/search"), - ("https://nitter.1d4.us/supernaturepics/search"), - ("https://nitter.kavin.rocks/id:2976459548/search"), - ("https://nitter.unixfox.eu/i/user/2976459548/search"), - ) + example = "https://nitter.net/USER/search" def tweets(self): return self._pagination("/search") @@ -386,95 +301,7 @@ class NitterTweetExtractor(NitterExtractor): filename_fmt = "{tweet_id}_{num}.{extension}" archive_fmt = "{tweet_id}_{num}" pattern = BASE_PATTERN + r"/(i/web|[^/?#]+)/status/(\d+())" - test = ( - ("https://nitter.net/supernaturepics/status/604341487988576256", { - "url": "3f2b64e175bf284aa672c3bb53ed275e470b919a", - "content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", - "keyword": { - "comments": 19, - "content": "Big Wedeene River, Canada", - "count": 1, - "date": "dt:2015-05-29 17:40:00", - "extension": "jpg", - "filename": "CGMNYZvW0AIVoom", - "likes": int, - "num": 1, - "quotes": 10, - "retweets": int, - "tweet_id": "604341487988576256", - "url": "https://nitter.net/pic/orig" - "/media%2FCGMNYZvW0AIVoom.jpg", - "user": { - "name": "supernaturepics", - "nick": "Nature Pictures", - }, - }, - }), - # 4 images - ("https://nitter.lacontrevoie.fr/i/status/894001459754180609", { - "url": "9c51b3a4a1114535eb9b168bba97ad95db0d59ff", - }), - # video - ("https://nitter.lacontrevoie.fr/i/status/1065692031626829824", { - "pattern": r"ytdl:https://nitter\.lacontrevoie\.fr/video" - r"/[0-9A-F]{10,}/https%3A%2F%2Fvideo.twimg.com%2F" - r"ext_tw_video%2F1065691868439007232%2Fpu%2Fpl%2F" - r"nv8hUQC1R0SjhzcZ.m3u8%3Ftag%3D5", - "keyword": { - "extension": "mp4", - "filename": "nv8hUQC1R0SjhzcZ", - }, - }), - # content with emoji, newlines, hashtags (#338) - ("https://nitter.1d4.us/playpokemon/status/1263832915173048321", { - "keyword": {"content": ( - r"re:Gear up for #PokemonSwordShieldEX with special Mystery " - "Gifts! \n\nYou’ll be able to receive four Galarian form " - "Pokémon with Hidden Abilities, plus some very useful items. " - "It’s our \\(Mystery\\) Gift to you, Trainers! \n\n❓🎁➡️ " - )}, - }), - # Nitter tweet (#890) - ("https://nitter.kavin.rocks/ed1conf/status/1163841619336007680", { - "url": "e115bd1c86c660064e392b05269bbcafcd8c8b7a", - "content": "f29501e44d88437fe460f5c927b7543fda0f6e34", - }), - # Reply to deleted tweet (#403, #838) - ("https://nitter.unixfox.eu/i/web/status/1170041925560258560", { - "pattern": r"https://nitter\.unixfox\.eu/pic/orig" - r"/media%2FEDzS7VrU0AAFL4_\.jpg", - }), - # "quoted" option (#854) - ("https://nitter.net/StobiesGalaxy/status/1270755918330896395", { - "options": (("quoted", True),), - "pattern": r"https://nitter\.net/pic/orig/media%2FEa[KG].+\.jpg", - "count": 8, - }), - # quoted tweet (#526, #854) - ("https://nitter.1d4.us/StobiesGalaxy/status/1270755918330896395", { - "pattern": r"https://nitter\.1d4\.us/pic/orig" - r"/enc/bWVkaWEvRWFL\w+LmpwZw==", - "keyword": {"filename": r"re:EaK.{12}"}, - "count": 4, - }), - # deleted quote tweet (#2225) - ("https://nitter.lacontrevoie.fr/i/status/1460044411165888515", { - "count": 0, - }), - # "Misleading" content - ("https://nitter.lacontrevoie.fr/i/status/1486373748911575046", { - "count": 4, - }), - # age-restricted (#2354) - ("https://nitter.unixfox.eu/mightbecurse/status/1492954264909479936", { - "keyword": {"date": "dt:2022-02-13 20:10:00"}, - "count": 1, - }), - # broadcast - ("https://nitter.it/POTUS/status/1639409307878928384", { - "count": 0, - }) - ) + example = "https://nitter.net/USER/status/12345" def tweets(self): url = "{}/i/status/{}".format(self.root, self.user) diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index af2a3678..8c7ffe52 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -24,11 +24,11 @@ class NozomiExtractor(Extractor): filename_fmt = "{postid} {dataid}.{extension}" archive_fmt = "{dataid}" - def items(self): + def _init(self): + self.session.headers["Origin"] = self.root + def items(self): data = self.metadata() - self.session.headers["Origin"] = self.root - self.session.headers["Referer"] = self.root + "/" for post_id in map(str, self.posts()): url = "https://j.nozomi.la/post/{}/{}/{}.json".format( @@ -105,52 +105,7 @@ class NozomiPostExtractor(NozomiExtractor): """Extractor for individual posts on nozomi.la""" subcategory = "post" pattern = r"(?:https?://)?nozomi\.la/post/(\d+)" - test = ( - ("https://nozomi.la/post/3649262.html", { - "url": "e5525e717aec712843be8b88592d6406ae9e60ba", - "pattern": r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb" - r"6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", - "content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", - "keyword": { - "artist" : ["hammer (sunset beach)"], - "character": ["patchouli knowledge"], - "copyright": ["touhou"], - "dataid" : "re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", - "date" : "dt:2016-07-26 02:32:03", - "extension": "webp", - "filename" : str, - "height" : 768, - "is_video" : False, - "postid" : 3649262, - "tags" : list, - "type" : "jpg", - "url" : str, - "width" : 1024, - }, - }), - # multiple images per post - ("https://nozomi.la/post/25588032.html", { - "url": "fb956ccedcf2cf509739d26e2609e910244aa56c", - "keyword": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", - "count": 7, - }), - # empty 'date' (#1163) - ("https://nozomi.la/post/130309.html", { - "keyword": {"date": None}, - }), - # gif - ("https://nozomi.la/post/1647.html", { - "pattern": r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c1" - r"49db459d76b58a074416c260ed93cc31fa9f0a\.gif", - "content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", - }), - # video - ("https://nozomi.la/post/2269847.html", { - "pattern": r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f" - r"2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", - "content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", - }), - ) + example = "https://nozomi.la/post/12345.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -165,11 +120,7 @@ class NozomiIndexExtractor(NozomiExtractor): subcategory = "index" pattern = (r"(?:https?://)?nozomi\.la/" r"(?:(index(?:-Popular)?)-(\d+)\.html)?(?:$|#|\?)") - test = ( - ("https://nozomi.la/"), - ("https://nozomi.la/index-2.html"), - ("https://nozomi.la/index-Popular-33.html"), - ) + example = "https://nozomi.la/index-1.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -183,11 +134,7 @@ class NozomiTagExtractor(NozomiExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{dataid}" pattern = r"(?:https?://)?nozomi\.la/tag/([^/?#]+)-(\d+)\." - test = ("https://nozomi.la/tag/3:1_aspect_ratio-1.html", { - "pattern": r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", - "count": ">= 25", - "range": "1-25", - }) + example = "https://nozomi.la/tag/TAG-1.html" def __init__(self, match): NozomiExtractor.__init__(self, match) @@ -205,9 +152,7 @@ class NozomiSearchExtractor(NozomiExtractor): directory_fmt = ("{category}", "{search_tags:J }") archive_fmt = "t_{search_tags}_{dataid}" pattern = r"(?:https?://)?nozomi\.la/search\.html\?q=([^&#]+)" - test = ("https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", { - "count": ">= 5", - }) + example = "https://nozomi.la/search.html?q=QUERY" def __init__(self, match): NozomiExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/nsfwalbum.py b/gallery_dl/extractor/nsfwalbum.py index 6433fbd2..7229bde5 100644 --- a/gallery_dl/extractor/nsfwalbum.py +++ b/gallery_dl/extractor/nsfwalbum.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,11 +21,7 @@ class NsfwalbumAlbumExtractor(GalleryExtractor): directory_fmt = ("{category}", "{album_id} {title}") archive_fmt = "{id}" pattern = r"(?:https?://)?(?:www\.)?nsfwalbum\.com(/album/(\d+))" - test = ("https://nsfwalbum.com/album/401611", { - "range": "1-5", - "url": "b0481fc7fad5982da397b6359fbed8421b8ba284", - "keyword": "e98f9b0d473c00000831618d0235863b1dd78294", - }) + example = "https://nsfwalbum.com/album/12345" def __init__(self, match): self.album_id = match.group(2) diff --git a/gallery_dl/extractor/nudecollect.py b/gallery_dl/extractor/nudecollect.py index 31599198..bda5d774 100644 --- a/gallery_dl/extractor/nudecollect.py +++ b/gallery_dl/extractor/nudecollect.py @@ -37,26 +37,8 @@ class NudecollectImageExtractor(NudecollectExtractor): pattern = (r"(?:https?://)?(?:www\.)?nudecollect\.com" r"(/content/([^/?#]+)/image-(\d+)-pics-(\d+)" r"-mirror-(\d+)\.html)") - test = ( - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/image-4-pics-108-mirror-43.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00004-5896498214-43" - r"-9689595623/20201220_Teenpornstorage_Patritcy_Vaness" - r"a_Lesbian_Lust/9879560327/nudecollect\.com\.jpg"), - "keyword": { - "slug" : ("20201220_Teenpornstorage_Patritcy" - "_Vanessa_Lesbian_Lust"), - "title" : ("20201220 Teenpornstorage Patritcy" - " Vanessa Lesbian Lust"), - "num" : 4, - "count" : 108, - "mirror": 43, - }, - }), - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/image-10-pics-108-mirror-43.html")), - ) + example = ("https://www.nudecollect.com/content/12345_TITLE" + "/image-1-pics-108-mirror-1.html") def __init__(self, match): NudecollectExtractor.__init__(self, match) @@ -80,45 +62,8 @@ class NudecollectAlbumExtractor(NudecollectExtractor): pattern = (r"(?:https?://)?(?:www\.)?nudecollect\.com" r"/content/([^/?#]+)/(?:index-mirror-(\d+)-(\d+)" r"|page-\d+-pics-(\d+)-mirror-(\d+))\.html") - test = ( - (("https://www.nudecollect.com/content/20170219_TheWhiteBoxxx_" - "Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex" - "_with_alluring_Czech_babes_x125_1080px/index-mirror-67-125.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00\d\d\d-5896498214-67" - r"-9689595623/20170219_TheWhiteBoxxx_Caprice" - r"_Tracy_Loves_Hot_ass_fingering_and_sensual_" - r"lesbian_sex_with_alluring_Czech_babes_x125_1080px" - r"/9879560327/nudecollect\.com\.jpg"), - "count" : 125, - "keyword": { - "slug" : ("20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_" - "ass_fingering_and_sensual_lesbian_sex_with_" - "alluring_Czech_babes_x125_1080px"), - "title" : ("20170219 TheWhiteBoxxx Caprice Tracy Loves Hot ass" - " fingering and sensual lesbian sex with alluring" - " Czech babes x125 1080px"), - "num" : int, - "mirror": 67, - }, - }), - (("https://www.nudecollect.com/content/20201220_Teenpornstorage_" - "Patritcy_Vanessa_Lesbian_Lust/page-1-pics-108-mirror-43.html"), { - "pattern": (r"https://mirror\d+\.nudecollect\.com/showimage" - r"/nudecollect-8769086487/image00\d\d\d-5896498214-43" - r"-9689595623/20201220_Teenpornstorage_Patritcy_Vaness" - r"a_Lesbian_Lust/9879560327/nudecollect\.com\.jpg"), - "count" : 108, - "keyword": { - "slug" : ("20201220_Teenpornstorage_Patritcy" - "_Vanessa_Lesbian_Lust"), - "title" : ("20201220 Teenpornstorage Patritcy" - " Vanessa Lesbian Lust"), - "num" : int, - "mirror": 43, - }, - }), - ) + example = ("https://www.nudecollect.com/content/12345_TITLE" + "/index-mirror-01-123.html") def __init__(self, match): self.slug = match.group(1) diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index f109d258..45313c51 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -243,6 +243,7 @@ class OAuthBase(Extractor): class OAuthFlickr(OAuthBase): subcategory = "flickr" pattern = "oauth:flickr$" + example = "oauth:flickr" redirect_uri = REDIRECT_URI_HTTPS def items(self): @@ -261,6 +262,7 @@ class OAuthFlickr(OAuthBase): class OAuthSmugmug(OAuthBase): subcategory = "smugmug" pattern = "oauth:smugmug$" + example = "oauth:smugmug" def items(self): yield Message.Version, 1 @@ -278,6 +280,7 @@ class OAuthSmugmug(OAuthBase): class OAuthTumblr(OAuthBase): subcategory = "tumblr" pattern = "oauth:tumblr$" + example = "oauth:tumblr" def items(self): yield Message.Version, 1 @@ -298,6 +301,7 @@ class OAuthTumblr(OAuthBase): class OAuthDeviantart(OAuthBase): subcategory = "deviantart" pattern = "oauth:deviantart$" + example = "oauth:deviantart" redirect_uri = REDIRECT_URI_HTTPS def items(self): @@ -319,6 +323,7 @@ class OAuthDeviantart(OAuthBase): class OAuthReddit(OAuthBase): subcategory = "reddit" pattern = "oauth:reddit$" + example = "oauth:reddit" def items(self): yield Message.Version, 1 @@ -340,6 +345,7 @@ class OAuthReddit(OAuthBase): class OAuthMastodon(OAuthBase): subcategory = "mastodon" pattern = "oauth:mastodon:(?:https?://)?([^/?#]+)" + example = "oauth:mastodon:mastodon.social" def __init__(self, match): OAuthBase.__init__(self, match) @@ -397,6 +403,7 @@ class OAuthMastodon(OAuthBase): class OAuthPixiv(OAuthBase): subcategory = "pixiv" pattern = "oauth:pixiv$" + example = "oauth:pixiv" def items(self): yield Message.Version, 1 diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py index 6bc7b9a9..0389ead9 100644 --- a/gallery_dl/extractor/paheal.py +++ b/gallery_dl/extractor/paheal.py @@ -74,34 +74,7 @@ class PahealTagExtractor(PahealExtractor): directory_fmt = ("{category}", "{search_tags}") pattern = (r"(?:https?://)?(?:rule34|rule63|cosplay)\.paheal\.net" r"/post/list/([^/?#]+)") - test = ( - ("https://rule34.paheal.net/post/list/Ayane_Suzuki/1", { - "pattern": r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20", - "count": ">= 15" - }), - ("https://rule34.paheal.net/post/list/Ayane_Suzuki/1", { - "range": "1", - "options": (("metadata", True),), - "keyword": { - "date": "dt:2018-01-07 07:04:05", - "duration": 0.0, - "extension": "jpg", - "filename": "2446128 - Ayane_Suzuki Idolmaster " - "idolmaster_dearly_stars Zanzi", - "height": 768, - "id": 2446128, - "md5": "b0ceda9d860df1d15b60293a7eb465c1", - "search_tags": "Ayane_Suzuki", - "size": 205312, - "source": "https://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=19957280", - "tags": "Ayane_Suzuki Idolmaster " - "idolmaster_dearly_stars Zanzi", - "uploader": "XXXname", - "width": 1024, - }, - }), - ) + example = "https://rule34.paheal.net/post/list/TAG/1" per_page = 70 def __init__(self, match): @@ -161,61 +134,7 @@ class PahealPostExtractor(PahealExtractor): subcategory = "post" pattern = (r"(?:https?://)?(?:rule34|rule63|cosplay)\.paheal\.net" r"/post/view/(\d+)") - test = ( - ("https://rule34.paheal.net/post/view/481609", { - "pattern": r"https://tulip\.paheal\.net/_images" - r"/bbdc1c33410c2cdce7556c7990be26b7/481609%20-%20" - r"Azumanga_Daioh%20inanimate%20Osaka%20Vuvuzela\.jpg", - "content": "7b924bcf150b352ac75c9d281d061e174c851a11", - "keyword": { - "date": "dt:2010-06-17 15:40:23", - "extension": "jpg", - "file_url": "re:https://tulip.paheal.net/_images/bbdc1c33410c", - "filename": "481609 - Azumanga_Daioh inanimate Osaka Vuvuzela", - "height": 660, - "id": 481609, - "md5": "bbdc1c33410c2cdce7556c7990be26b7", - "size": 157389, - "source": "", - "tags": "Azumanga_Daioh inanimate Osaka Vuvuzela", - "uploader": "CaptainButtface", - "width": 614, - }, - }), - ("https://rule34.paheal.net/post/view/488534", { - "keyword": { - "date": "dt:2010-06-25 13:51:17", - "height": 800, - "md5": "b39edfe455a0381110c710d6ed2ef57d", - "size": 758989, - "source": "http://www.furaffinity.net/view/4057821/", - "tags": "inanimate thelost-dragon Vuvuzela", - "uploader": "leacheate_soup", - "width": 1200, - }, - }), - # video - ("https://rule34.paheal.net/post/view/3864982", { - "pattern": r"https://[\w]+\.paheal\.net/_images/7629fc0ff77e32637d" - r"de5bf4f992b2cb/3864982%20-%20animated%20Metal_Gear%20" - r"Metal_Gear_Solid_V%20Quiet%20Vg_erotica%20webm\.webm", - "keyword": { - "date": "dt:2020-09-06 01:59:03", - "duration": 30.0, - "extension": "webm", - "height": 2500, - "id": 3864982, - "md5": "7629fc0ff77e32637dde5bf4f992b2cb", - "size": 18454938, - "source": "https://twitter.com/VG_Worklog" - "/status/1302407696294055936", - "tags": "animated Metal_Gear Metal_Gear_Solid_V " - "Quiet Vg_erotica webm", - "uploader": "justausername", - "width": 1768, - }, - }), - ) + example = "https://rule34.paheal.net/post/view/12345" def __init__(self, match): PahealExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 99d9457a..729ceaf1 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -103,7 +103,6 @@ class PatreonExtractor(Extractor): def _pagination(self, url): headers = { - "Referer" : self.root + "/", "Content-Type": "application/vnd.api+json", } @@ -251,7 +250,7 @@ class PatreonExtractor(Extractor): def _extract_bootstrap(self, page): return util.json_loads(text.extr( - page, "window.patreon.bootstrap,", "\n});") + "}") + page, "window.patreon.bootstrap,", "});") + "}") class PatreonCreatorExtractor(PatreonExtractor): @@ -260,34 +259,7 @@ class PatreonCreatorExtractor(PatreonExtractor): pattern = (r"(?:https?://)?(?:www\.)?patreon\.com" r"/(?!(?:home|join|posts|login|signup)(?:$|[/?#]))" r"([^/?#]+)(?:/posts)?/?(?:\?([^#]+))?") - test = ( - ("https://www.patreon.com/koveliana", { - "range": "1-25", - "count": ">= 25", - "keyword": { - "attachments" : list, - "comment_count": int, - "content" : str, - "creator" : dict, - "date" : "type:datetime", - "id" : int, - "images" : list, - "like_count" : int, - "post_type" : str, - "published_at" : str, - "title" : str, - }, - }), - ("https://www.patreon.com/koveliana/posts?filters[month]=2020-3", { - "count": 1, - "keyword": {"date": "dt:2020-03-30 21:21:44"}, - }), - ("https://www.patreon.com/kovelianot", { - "exception": exception.NotFoundError, - }), - ("https://www.patreon.com/user?u=2931440"), - ("https://www.patreon.com/user/posts/?u=2931440"), - ) + example = "https://www.patreon.com/USER" def __init__(self, match): PatreonExtractor.__init__(self, match) @@ -328,7 +300,7 @@ class PatreonUserExtractor(PatreonExtractor): """Extractor for media from creators supported by you""" subcategory = "user" pattern = r"(?:https?://)?(?:www\.)?patreon\.com/home$" - test = ("https://www.patreon.com/home",) + example = "https://www.patreon.com/home" def posts(self): url = self._build_url("stream", ( @@ -343,24 +315,7 @@ class PatreonPostExtractor(PatreonExtractor): """Extractor for media from a single post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?patreon\.com/posts/([^/?#]+)" - test = ( - # postfile + attachments - ("https://www.patreon.com/posts/precious-metal-23563293", { - "count": 4, - }), - # postfile + content - ("https://www.patreon.com/posts/56127163", { - "count": 3, - "keyword": {"filename": r"re:^(?!1).+$"}, - }), - # tags (#1539) - ("https://www.patreon.com/posts/free-post-12497641", { - "keyword": {"tags": ["AWMedia"]}, - }), - ("https://www.patreon.com/posts/not-found-123", { - "exception": exception.NotFoundError, - }), - ) + example = "https://www.patreon.com/posts/TITLE-12345" def __init__(self, match): PatreonExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index 8fa5de24..3a0f5b02 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -57,68 +57,7 @@ class PhilomenaPostExtractor(PhilomenaExtractor): """Extractor for single posts on a Philomena booru""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:images/)?(\d+)" - test = ( - ("https://derpibooru.org/images/1", { - "content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", - "count": 1, - "keyword": { - "animated": False, - "aspect_ratio": 1.0, - "comment_count": int, - "created_at": "2012-01-02T03:12:33Z", - "date": "dt:2012-01-02 03:12:33", - "deletion_reason": None, - "description": "", - "downvotes": int, - "duplicate_of": None, - "duration": 0.04, - "extension": "png", - "faves": int, - "first_seen_at": "2012-01-02T03:12:33Z", - "format": "png", - "height": 900, - "hidden_from_users": False, - "id": 1, - "mime_type": "image/png", - "name": "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+ga" - "lore_artist-colon-speccysy_get_sunshine", - "orig_sha512_hash": None, - "processed": True, - "representations": dict, - "score": int, - "sha512_hash": "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391" - "aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07e" - "f7800663b994f3f343c572da5ecc22a9660f", - "size": 860914, - "source_url": "https://www.deviantart.com/speccysy/art" - "/Afternoon-Flight-215193985", - "spoilered": False, - "tag_count": int, - "tag_ids": list, - "tags": list, - "thumbnails_generated": True, - "updated_at": r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", - "uploader": "Clover the Clever", - "uploader_id": 211188, - "upvotes": int, - "view_url": str, - "width": 900, - "wilson_score": float, - }, - }), - ("https://derpibooru.org/1"), - ("https://www.derpibooru.org/1"), - ("https://www.derpibooru.org/images/1"), - - ("https://ponybooru.org/images/1", { - "content": "bca26f58fafd791fe07adcd2a28efd7751824605", - }), - ("https://www.ponybooru.org/images/1"), - - ("https://furbooru.org/images/1", { - "content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", - }), - ) + example = "https://derpibooru.org/images/12345" def __init__(self, match): PhilomenaExtractor.__init__(self, match) @@ -133,28 +72,7 @@ class PhilomenaSearchExtractor(PhilomenaExtractor): subcategory = "search" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?:search/?\?([^#]+)|tags/([^/?#]+))" - test = ( - ("https://derpibooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://derpibooru.org/tags/cute", { - "range": "40-60", - "count": 21, - }), - (("https://derpibooru.org/tags/" - "artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak"), { - "count": ">= 2", - }), - ("https://ponybooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://furbooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ) + example = "https://derpibooru.org/search?q=QUERY" def __init__(self, match): PhilomenaExtractor.__init__(self, match) @@ -188,28 +106,7 @@ class PhilomenaGalleryExtractor(PhilomenaExtractor): directory_fmt = ("{category}", "galleries", "{gallery[id]} {gallery[title]}") pattern = BASE_PATTERN + r"/galleries/(\d+)" - test = ( - ("https://derpibooru.org/galleries/1", { - "pattern": r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", - "keyword": { - "gallery": { - "description": "Indexes start at 1 :P", - "id": 1, - "spoiler_warning": "", - "thumbnail_id": 1, - "title": "The Very First Gallery", - "user": "DeliciousBlackInk", - "user_id": 365446, - }, - }, - }), - ("https://ponybooru.org/galleries/27", { - "count": ">= 24", - }), - ("https://furbooru.org/galleries/27", { - "count": ">= 13", - }), - ) + example = "https://derpibooru.org/galleries/12345" def __init__(self, match): PhilomenaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/photobucket.py b/gallery_dl/extractor/photobucket.py index 22aff884..a01c9fe6 100644 --- a/gallery_dl/extractor/photobucket.py +++ b/gallery_dl/extractor/photobucket.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://photobucket.com/""" +"""Extractors for https://photobucket.com/""" from .common import Extractor, Message from .. import text, exception @@ -23,29 +23,7 @@ class PhotobucketAlbumExtractor(Extractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?((?:[\w-]+\.)?photobucket\.com)" r"/user/[^/?&#]+/library(?:/[^?&#]*)?") - test = ( - ("https://s369.photobucket.com/user/CrpyLrkr/library", { - "pattern": r"https?://[oi]+\d+.photobucket.com/albums/oo139/", - "count": ">= 50" - }), - # subalbums of main "directory" - ("https://s271.photobucket.com/user/lakerfanryan/library/", { - "options": (("image-filter", "False"),), - "pattern": pattern, - "count": 1, - }), - # subalbums of subalbum without images - ("https://s271.photobucket.com/user/lakerfanryan/library/Basketball", { - "pattern": pattern, - "count": ">= 9", - }), - # private (missing JSON data) - ("https://s1277.photobucket.com/user/sinisterkat44/library/", { - "count": 0, - }), - ("https://s1110.photobucket.com/user/chndrmhn100/library/" - "Chandu%20is%20the%20King?sort=3&page=1"), - ) + example = "https://s123.photobucket.com/user/USER/library" def __init__(self, match): self.root = "https://" + match.group(1) @@ -113,18 +91,7 @@ class PhotobucketImageExtractor(Extractor): pattern = (r"(?:https?://)?(?:[\w-]+\.)?photobucket\.com" r"(?:/gallery/user/([^/?&#]+)/media/([^/?&#]+)" r"|/user/([^/?&#]+)/media/[^?&#]+\.html)") - test = ( - (("https://s271.photobucket.com/user/lakerfanryan" - "/media/Untitled-3-1.jpg.html"), { - "url": "3b647deeaffc184cc48c89945f67574559c9051f", - "keyword": "69732741b2b351db7ecaa77ace2fdb39f08ca5a3", - }), - (("https://s271.photobucket.com/user/lakerfanryan" - "/media/IsotopeswBros.jpg.html?sort=3&o=2"), { - "url": "12c1890c09c9cdb8a88fba7eec13f324796a8d7b", - "keyword": "61200a223df6c06f45ac3d30c88b3f5b048ce9a8", - }), - ) + example = "https://s123.photobucket.com/user/USER/media/NAME.EXT.html" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/photovogue.py b/gallery_dl/extractor/photovogue.py index 3c68fd51..2a2df5a6 100644 --- a/gallery_dl/extractor/photovogue.py +++ b/gallery_dl/extractor/photovogue.py @@ -19,38 +19,7 @@ class PhotovogueUserExtractor(Extractor): filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/photographers/(\d+)" - test = ( - ("https://www.vogue.com/photovogue/photographers/221252"), - ("https://vogue.com/photovogue/photographers/221252", { - "pattern": r"https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - "keyword": { - "date": "type:datetime", - "favorite_count": int, - "favorited": list, - "id": int, - "image_id": str, - "is_favorite": False, - "orientation": "re:portrait|landscape", - "photographer": { - "biography": "Born in 1995. Live in Bologna.", - "city": "Bologna", - "country_id": 106, - "favoritedCount": int, - "id": 221252, - "isGold": bool, - "isPro": bool, - "latitude": str, - "longitude": str, - "name": "Arianna Mattarozzi", - "user_id": "38cb0601-4a85-453c-b7dc-7650a037f2ab", - "websites": list, - }, - "photographer_id": 221252, - "tags": list, - "title": str, - }, - }), - ) + example = "https://www.vogue.com/photovogue/photographers/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/picarto.py b/gallery_dl/extractor/picarto.py index 77a07b48..cc7eee5f 100644 --- a/gallery_dl/extractor/picarto.py +++ b/gallery_dl/extractor/picarto.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,12 +21,7 @@ class PicartoGalleryExtractor(Extractor): filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?picarto\.tv/([^/?#]+)/gallery" - test = ("https://picarto.tv/fnook/gallery/default/", { - "pattern": r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork" - r"/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", - "count": ">= 7", - "keyword": {"date": "type:datetime"}, - }) + example = "https://picarto.tv/USER/gallery/TITLE/" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/piczel.py b/gallery_dl/extractor/piczel.py index 56c29782..422325ff 100644 --- a/gallery_dl/extractor/piczel.py +++ b/gallery_dl/extractor/piczel.py @@ -68,10 +68,7 @@ class PiczelUserExtractor(PiczelExtractor): """Extractor for all images from a user's gallery""" subcategory = "user" pattern = r"(?:https?://)?(?:www\.)?piczel\.tv/gallery/([^/?#]+)/?$" - test = ("https://piczel.tv/gallery/Bikupan", { - "range": "1-100", - "count": ">= 100", - }) + example = "https://piczel.tv/gallery/USER" def __init__(self, match): PiczelExtractor.__init__(self, match) @@ -89,9 +86,7 @@ class PiczelFolderExtractor(PiczelExtractor): archive_fmt = "f{folder[id]}_{id}_{num}" pattern = (r"(?:https?://)?(?:www\.)?piczel\.tv" r"/gallery/(?!image)([^/?#]+)/(\d+)") - test = ("https://piczel.tv/gallery/Lulena/1114", { - "count": ">= 4", - }) + example = "https://piczel.tv/gallery/USER/12345" def __init__(self, match): PiczelExtractor.__init__(self, match) @@ -106,30 +101,7 @@ class PiczelImageExtractor(PiczelExtractor): """Extractor for individual images""" subcategory = "image" pattern = r"(?:https?://)?(?:www\.)?piczel\.tv/gallery/image/(\d+)" - test = ("https://piczel.tv/gallery/image/7807", { - "pattern": r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image" - r"/32920/image/7807/1532236438-Lulena\.png", - "content": "df9a053a24234474a19bce2b7e27e0dec23bff87", - "keyword": { - "created_at": "2018-07-22T05:13:58.000Z", - "date": "dt:2018-07-22 05:13:58", - "description": None, - "extension": "png", - "favorites_count": int, - "folder_id": 1113, - "id": 7807, - "is_flash": False, - "is_video": False, - "multi": False, - "nsfw": False, - "num": 0, - "password_protected": False, - "tags": ["fanart", "commission", "altair", "recreators"], - "title": "Altair", - "user": dict, - "views": int, - }, - }) + example = "https://piczel.tv/gallery/image/12345" def __init__(self, match): PiczelExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pillowfort.py b/gallery_dl/extractor/pillowfort.py index af7d57f1..8cfefa97 100644 --- a/gallery_dl/extractor/pillowfort.py +++ b/gallery_dl/extractor/pillowfort.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -122,69 +122,7 @@ class PillowfortPostExtractor(PillowfortExtractor): """Extractor for a single pillowfort post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.pillowfort.social/posts/27510", { - "pattern": r"https://img\d+\.pillowfort\.social" - r"/posts/\w+_out\d+\.png", - "count": 4, - "keyword": { - "avatar_url": str, - "col": 0, - "commentable": True, - "comments_count": int, - "community_id": None, - "content": str, - "created_at": str, - "date": "type:datetime", - "deleted": None, - "deleted_at": None, - "deleted_by_mod": None, - "deleted_for_flag_id": None, - "embed_code": None, - "id": int, - "last_activity": str, - "last_activity_elapsed": str, - "last_edited_at": str, - "likes_count": int, - "media_type": "picture", - "nsfw": False, - "num": int, - "original_post_id": None, - "original_post_user_id": None, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id": 27510, - "post_type": "picture", - "privacy": "public", - "reblog_copy_info": list, - "rebloggable": True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count": int, - "row": int, - "small_image_url": None, - "tags": list, - "time_elapsed": str, - "timestamp": str, - "title": "What is Pillowfort.social?", - "updated_at": str, - "url": r"re:https://img3.pillowfort.social/posts/.*\.png", - "user_id": 5, - "username": "Staff" - }, - }), - ("https://www.pillowfort.social/posts/1557500", { - "options": (("external", True), ("inline", False)), - "pattern": r"https://twitter\.com/Aliciawitdaart/status" - r"/1282862493841457152", - }), - ("https://www.pillowfort.social/posts/1672518", { - "options": (("inline", True),), - "count": 3, - }), - ) + example = "https://www.pillowfort.social/posts/12345" def posts(self): url = "{}/posts/{}/json/".format(self.root, self.item) @@ -195,11 +133,7 @@ class PillowfortUserExtractor(PillowfortExtractor): """Extractor for all posts of a pillowfort user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!posts/)([^/?#]+)" - test = ("https://www.pillowfort.social/Pome", { - "pattern": r"https://img\d+\.pillowfort\.social/posts/", - "range": "1-15", - "count": 15, - }) + example = "https://www.pillowfort.social/USER" def posts(self): url = "{}/{}/json/".format(self.root, self.item) diff --git a/gallery_dl/extractor/pinterest.py b/gallery_dl/extractor/pinterest.py index be30705b..e9f124f1 100644 --- a/gallery_dl/extractor/pinterest.py +++ b/gallery_dl/extractor/pinterest.py @@ -111,21 +111,7 @@ class PinterestPinExtractor(PinterestExtractor): """Extractor for images from a single pin from pinterest.com""" subcategory = "pin" pattern = BASE_PATTERN + r"/pin/([^/?#]+)(?!.*#related$)" - test = ( - ("https://www.pinterest.com/pin/858146903966145189/", { - "url": "afb3c26719e3a530bb0e871c480882a801a4e8a5", - "content": ("4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", - "d3e24bc9f7af585e8c23b9136956bd45a4d9b947"), - }), - # video pin (#1189) - ("https://www.pinterest.com/pin/422564377542934214/", { - "pattern": r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff" - r"/d722ff00ab2352981b89974b37909de8.m3u8", - }), - ("https://www.pinterest.com/pin/858146903966145188/", { - "exception": exception.NotFoundError, - }), - ) + example = "https://www.pinterest.com/pin/12345/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -147,26 +133,7 @@ class PinterestBoardExtractor(PinterestExtractor): archive_fmt = "{board[id]}_{id}" pattern = (BASE_PATTERN + r"/(?!pin/)([^/?#]+)" "/(?!_saved|_created|pins/)([^/?#]+)/?$") - test = ( - ("https://www.pinterest.com/g1952849/test-/", { - "pattern": r"https://i\.pinimg\.com/originals/", - "count": 2, - }), - # board with sections (#835) - ("https://www.pinterest.com/g1952849/stuff/", { - "options": (("sections", True),), - "count": 4, - }), - # secret board (#1055) - ("https://www.pinterest.de/g1952849/secret/", { - "count": 2, - }), - ("https://www.pinterest.com/g1952848/test/", { - "exception": exception.GalleryDLException, - }), - # .co.uk TLD (#914) - ("https://www.pinterest.co.uk/hextra7519/based-animals/"), - ) + example = "https://www.pinterest.com/USER/BOARD/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -197,13 +164,7 @@ class PinterestUserExtractor(PinterestExtractor): """Extractor for a user's boards""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)(?:/_saved)?/?$" - test = ( - ("https://www.pinterest.com/g1952849/", { - "pattern": PinterestBoardExtractor.pattern, - "count": ">= 2", - }), - ("https://www.pinterest.com/g1952849/_saved/"), - ) + example = "https://www.pinterest.com/USER/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -222,11 +183,7 @@ class PinterestAllpinsExtractor(PinterestExtractor): subcategory = "allpins" directory_fmt = ("{category}", "{user}") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/pins/?$" - test = ("https://www.pinterest.com/g1952849/pins/", { - "pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", - "count": 7, - }) + example = "https://www.pinterest.com/USER/pins/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -244,12 +201,7 @@ class PinterestCreatedExtractor(PinterestExtractor): subcategory = "created" directory_fmt = ("{category}", "{user}") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/_created/?$" - test = ("https://www.pinterest.de/digitalmomblog/_created/", { - "pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}" - r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png)", - "count": 10, - "range": "1-10", - }) + example = "https://www.pinterest.com/USER/_created/" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -269,9 +221,7 @@ class PinterestSectionExtractor(PinterestExtractor): "{board[name]}", "{section[title]}") archive_fmt = "{board[id]}_{id}" pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/([^/?#]+)/([^/?#]+)" - test = ("https://www.pinterest.com/g1952849/stuff/section", { - "count": 2, - }) + example = "https://www.pinterest.com/USER/BOARD/SEcTION" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -299,10 +249,7 @@ class PinterestSearchExtractor(PinterestExtractor): subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search/pins/?\?q=([^&#]+)" - test = ("https://www.pinterest.com/search/pins/?q=nature", { - "range": "1-50", - "count": ">= 50", - }) + example = "https://www.pinterest.com/search/pins/?q=QUERY" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -320,11 +267,7 @@ class PinterestRelatedPinExtractor(PinterestPinExtractor): subcategory = "related-pin" directory_fmt = ("{category}", "related {original_pin[id]}") pattern = BASE_PATTERN + r"/pin/([^/?#]+).*#related$" - test = ("https://www.pinterest.com/pin/858146903966145189/#related", { - "range": "31-70", - "count": 40, - "archive": False, - }) + example = "https://www.pinterest.com/pin/12345/#related" def metadata(self): return {"original_pin": self.api.pin(self.pin_id)} @@ -339,11 +282,7 @@ class PinterestRelatedBoardExtractor(PinterestBoardExtractor): directory_fmt = ("{category}", "{board[owner][username]}", "{board[name]}", "related") pattern = BASE_PATTERN + r"/(?!pin/)([^/?#]+)/([^/?#]+)/?#related$" - test = ("https://www.pinterest.com/g1952849/test-/#related", { - "range": "31-70", - "count": 40, - "archive": False, - }) + example = "https://www.pinterest.com/USER/BOARD/#related" def pins(self): return self.api.board_content_recommendation(self.board["id"]) @@ -353,15 +292,7 @@ class PinterestPinitExtractor(PinterestExtractor): """Extractor for images from a pin.it URL""" subcategory = "pinit" pattern = r"(?:https?://)?pin\.it/([^/?#]+)" - - test = ( - ("https://pin.it/Hvt8hgT", { - "url": "8daad8558382c68f0868bdbd17d05205184632fa", - }), - ("https://pin.it/Hvt8hgS", { - "exception": exception.NotFoundError, - }), - ) + example = "https://pin.it/abcde" def __init__(self, match): PinterestExtractor.__init__(self, match) @@ -394,7 +325,6 @@ class PinterestAPI(): "Accept" : "application/json, text/javascript, " "*/*, q=0.01", "Accept-Language" : "en-US,en;q=0.5", - "Referer" : self.root + "/", "X-Requested-With" : "XMLHttpRequest", "X-APP-VERSION" : "0c4af40", "X-CSRFToken" : csrf_token, diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index ffe8030f..18a3ceb6 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -161,13 +161,7 @@ class PixivUserExtractor(PixivExtractor): pattern = (BASE_PATTERN + r"/(?:" r"(?:en/)?u(?:sers)?/|member\.php\?id=|(?:mypage\.php)?#id=" r")(\d+)(?:$|[?#])") - test = ( - ("https://www.pixiv.net/en/users/173530"), - ("https://www.pixiv.net/u/173530"), - ("https://www.pixiv.net/member.php?id=173530"), - ("https://www.pixiv.net/mypage.php#id=173530"), - ("https://www.pixiv.net/#id=173530"), - ) + example = "https://www.pixiv.net/en/users/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -195,29 +189,7 @@ class PixivArtworksExtractor(PixivExtractor): r"(?:en/)?users/(\d+)/(?:artworks|illustrations|manga)" r"(?:/([^/?#]+))?/?(?:$|[?#])" r"|member_illust\.php\?id=(\d+)(?:&([^#]+))?)") - test = ( - ("https://www.pixiv.net/en/users/173530/artworks", { - "url": "852c31ad83b6840bacbce824d85f2a997889efb7", - }), - # illusts with specific tag - (("https://www.pixiv.net/en/users/173530/artworks" - "/%E6%89%8B%E3%81%B6%E3%82%8D"), { - "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", - }), - (("https://www.pixiv.net/member_illust.php?id=173530" - "&tag=%E6%89%8B%E3%81%B6%E3%82%8D"), { - "url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", - }), - # deleted account - ("http://www.pixiv.net/member_illust.php?id=173531", { - "options": (("metadata", True),), - "exception": exception.NotFoundError, - }), - ("https://www.pixiv.net/en/users/173530/manga"), - ("https://www.pixiv.net/en/users/173530/illustrations"), - ("https://www.pixiv.net/member_illust.php?id=173530"), - ("https://touch.pixiv.net/member_illust.php?id=173530"), - ) + example = "https://www.pixiv.net/en/users/12345/artworks" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -253,9 +225,7 @@ class PixivAvatarExtractor(PixivExtractor): filename_fmt = "avatar{date:?_//%Y-%m-%d}.{extension}" archive_fmt = "avatar_{user[id]}_{date}" pattern = USER_PATTERN + r"/avatar" - test = ("https://www.pixiv.net/en/users/173530/avatar", { - "content": "4e57544480cc2036ea9608103e8f024fa737fe66", - }) + example = "https://www.pixiv.net/en/users/12345/avatar" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -273,10 +243,7 @@ class PixivBackgroundExtractor(PixivExtractor): filename_fmt = "background{date:?_//%Y-%m-%d}.{extension}" archive_fmt = "background_{user[id]}_{date}" pattern = USER_PATTERN + "/background" - test = ("https://www.pixiv.net/en/users/194921/background", { - "pattern": r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02" - r"/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", - }) + example = "https://www.pixiv.net/en/users/12345/background" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -303,14 +270,7 @@ class PixivMeExtractor(PixivExtractor): """Extractor for pixiv.me URLs""" subcategory = "me" pattern = r"(?:https?://)?pixiv\.me/([^/?#]+)" - test = ( - ("https://pixiv.me/del_shannon", { - "url": "29c295ce75150177e6b0a09089a949804c708fbf", - }), - ("https://pixiv.me/del_shanno", { - "exception": exception.NotFoundError, - }), - ) + example = "https://pixiv.me/USER" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -333,45 +293,7 @@ class PixivWorkExtractor(PixivExtractor): r"|(?:i(?:\d+\.pixiv|\.pximg)\.net" r"/(?:(?:.*/)?img-[^/]+/img/\d{4}(?:/\d\d){5}|img\d+/img/[^/]+)" r"|img\d*\.pixiv\.net/img/[^/]+|(?:www\.)?pixiv\.net/i)/(\d+))") - test = ( - ("https://www.pixiv.net/artworks/966412", { - "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", - "keyword": { - "date" : "dt:2008-06-12 15:29:13", - "date_url": "dt:2008-06-12 15:29:13", - }, - }), - (("http://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=966411"), { - "exception": exception.NotFoundError, - }), - # ugoira - (("https://www.pixiv.net/member_illust.php" - "?mode=medium&illust_id=66806629"), { - "url": "7267695a985c4db8759bebcf8d21dbdd2d2317ef", - "keyword": { - "frames" : list, - "date" : "dt:2018-01-14 15:06:08", - "date_url": "dt:2018-01-15 04:24:48", - }, - }), - # related works (#1237) - ("https://www.pixiv.net/artworks/966412", { - "options": (("related", True),), - "range": "1-10", - "count": ">= 10", - }), - ("https://www.pixiv.net/en/artworks/966412"), - ("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641"), - ("http://i1.pixiv.net/c/600x600/img-master" - "/img/2008/06/13/00/29/13/966412_p0_master1200.jpg"), - ("https://i.pximg.net/img-original" - "/img/2017/04/25/07/33/29/62568267_p0.png"), - ("https://www.pixiv.net/i/966412"), - ("http://img.pixiv.net/img/soundcross/42626136.jpg"), - ("http://i2.pixiv.net/img76/img/snailrin/42672235.jpg"), - ) + example = "https://www.pixiv.net/artworks/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -394,47 +316,7 @@ class PixivFavoriteExtractor(PixivExtractor): pattern = (BASE_PATTERN + r"/(?:(?:en/)?" r"users/(\d+)/(bookmarks/artworks|following)(?:/([^/?#]+))?" r"|bookmark\.php)(?:\?([^#]*))?") - test = ( - ("https://www.pixiv.net/en/users/173530/bookmarks/artworks", { - "url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", - }), - ("https://www.pixiv.net/bookmark.php?id=173530", { - "url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", - }), - # bookmarks with specific tag - (("https://www.pixiv.net/en/users/3137110" - "/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93"), { - "url": "379b28275f786d946e01f721e54afe346c148a8c", - }), - # bookmarks with specific tag (legacy url) - (("https://www.pixiv.net/bookmark.php?id=3137110" - "&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1"), { - "url": "379b28275f786d946e01f721e54afe346c148a8c", - }), - # own bookmarks - ("https://www.pixiv.net/bookmark.php", { - "url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "keyword": {"tags_bookmark": ["47", "hitman"]}, - "options": (("metadata-bookmark", True),), - }), - # own bookmarks with tag (#596) - ("https://www.pixiv.net/bookmark.php?tag=foobar", { - "count": 0, - }), - # followed users (#515) - ("https://www.pixiv.net/en/users/173530/following", { - "pattern": PixivUserExtractor.pattern, - "count": ">= 12", - }), - # followed users (legacy url) (#515) - ("https://www.pixiv.net/bookmark.php?id=173530&type=user", { - "pattern": PixivUserExtractor.pattern, - "count": ">= 12", - }), - # touch URLs - ("https://touch.pixiv.net/bookmark.php?id=173530"), - ("https://touch.pixiv.net/bookmark.php"), - ) + example = "https://www.pixiv.net/en/users/12345/bookmarks/artworks" def __init__(self, match): uid, kind, self.tag, query = match.groups() @@ -495,14 +377,7 @@ class PixivRankingExtractor(PixivExtractor): directory_fmt = ("{category}", "rankings", "{ranking[mode]}", "{ranking[date]}") pattern = BASE_PATTERN + r"/ranking\.php(?:\?([^#]*))?" - test = ( - ("https://www.pixiv.net/ranking.php?mode=daily&date=20170818"), - ("https://www.pixiv.net/ranking.php"), - ("https://touch.pixiv.net/ranking.php"), - ("https://www.pixiv.net/ranking.php?mode=unknown", { - "exception": exception.StopExtraction, - }), - ) + example = "https://www.pixiv.net/ranking.php" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -561,24 +436,7 @@ class PixivSearchExtractor(PixivExtractor): directory_fmt = ("{category}", "search", "{search[word]}") pattern = (BASE_PATTERN + r"/(?:(?:en/)?tags/([^/?#]+)(?:/[^/?#]+)?/?" r"|search\.php)(?:\?([^#]+))?") - test = ( - ("https://www.pixiv.net/en/tags/Original", { - "range": "1-10", - "count": 10, - }), - ("https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", { - "exception": exception.StopExtraction, - }), - ("https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", { - "exception": exception.StopExtraction, - }), - ("https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", { - "exception": exception.StopExtraction, - }), - ("https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag"), - ("https://www.pixiv.net/search.php?s_mode=s_tag&word=Original"), - ("https://touch.pixiv.net/search.php?word=Original"), - ) + example = "https://www.pixiv.net/en/tags/TAG" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -643,10 +501,7 @@ class PixivFollowExtractor(PixivExtractor): archive_fmt = "F_{user_follow[id]}_{id}{num}.{extension}" directory_fmt = ("{category}", "following") pattern = BASE_PATTERN + r"/bookmark_new_illust\.php" - test = ( - ("https://www.pixiv.net/bookmark_new_illust.php"), - ("https://touch.pixiv.net/bookmark_new_illust.php"), - ) + example = "https://www.pixiv.net/bookmark_new_illust.php" def works(self): return self.api.illust_follow() @@ -663,17 +518,7 @@ class PixivPixivisionExtractor(PixivExtractor): "{pixivision_id} {pixivision_title}") archive_fmt = "V{pixivision_id}_{id}{suffix}.{extension}" pattern = r"(?:https?://)?(?:www\.)?pixivision\.net/(?:en/)?a/(\d+)" - test = ( - ("https://www.pixivision.net/en/a/2791"), - ("https://pixivision.net/a/2791", { - "count": 7, - "keyword": { - "pixivision_id": "2791", - "pixivision_title": "What's your favorite music? Editor’s " - "picks featuring: “CD Covers”!", - }, - }), - ) + example = "https://www.pixivision.net/en/a/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -705,22 +550,7 @@ class PixivSeriesExtractor(PixivExtractor): "{series[id]} {series[title]}") filename_fmt = "{num_series:>03}_{id}_p{num}.{extension}" pattern = BASE_PATTERN + r"/user/(\d+)/series/(\d+)" - test = ("https://www.pixiv.net/user/10509347/series/21859", { - "range": "1-10", - "count": 10, - "keyword": { - "num_series": int, - "series": { - "canonical": "https://www.pixiv.net/user/10509347" - "/series/21859", - "description": str, - "ogp": dict, - "title": "先輩がうざい後輩の話", - "total": int, - "twitter": dict, - }, - }, - }) + example = "https://www.pixiv.net/user/12345/series/12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -762,56 +592,7 @@ class PixivNovelExtractor(PixivExtractor): subcategory = "novel" request_interval = 1.0 pattern = BASE_PATTERN + r"/n(?:ovel/show\.php\?id=|/)(\d+)" - test = ( - ("https://www.pixiv.net/novel/show.php?id=19612040", { - "count": 1, - "content": "8c818474153cbd2f221ee08766e1d634c821d8b4", - "keyword": { - "caption": r"re:「無能な名無し」と呼ばれ虐げられて育った鈴\(すず\)は、", - "comment_access_control": 0, - "create_date": "2023-04-02T15:18:58+09:00", - "date": "dt:2023-04-02 06:18:58", - "id": 19612040, - "is_bookmarked": False, - "is_muted": False, - "is_mypixiv_only": False, - "is_original": True, - "is_x_restricted": False, - "novel_ai_type": 1, - "page_count": 1, - "rating": "General", - "restrict": 0, - "series": { - "id": 10278364, - "title": "龍の贄嫁〜無能な名無しと虐げられていましたが、" - "どうやら異母妹に霊力を搾取されていたようです〜", - }, - "tags": ["和風ファンタジー", "溺愛", "神様", "ヤンデレ", "執着", - "異能", "ざまぁ", "学園", "神嫁"], - "text_length": 5974, - "title": "異母妹から「無能な名無し」と虐げられていた私、" - "どうやら異母妹に霊力を搾取されていたようです（１）", - "user": { - "account": "yukinaga_chifuyu", - "id": 77055466, - }, - "visible": True, - "x_restrict": 0, - }, - }), - # embeds - ("https://www.pixiv.net/novel/show.php?id=16422450", { - "options": (("embeds", True),), - "count": 3, - }), - # full series - ("https://www.pixiv.net/novel/show.php?id=19612040", { - "options": (("full-series", True),), - "count": 4, - }), - # short URL - ("https://www.pixiv.net/n/19612040"), - ) + example = "https://www.pixiv.net/novel/show.php?id=12345" def __init__(self, match): PixivExtractor.__init__(self, match) @@ -864,8 +645,13 @@ class PixivNovelExtractor(PixivExtractor): yield Message.Directory, novel + try: + content = self.api.novel_text(novel["id"])["novel_text"] + except Exception: + self.log.warning("Unable to download novel %s", novel["id"]) + continue + novel["extension"] = "txt" - content = self.api.novel_text(novel["id"])["novel_text"] yield Message.Url, "text:" + content, novel if embeds: @@ -917,11 +703,7 @@ class PixivNovelUserExtractor(PixivNovelExtractor): """Extractor for pixiv users' novels""" subcategory = "novel-user" pattern = USER_PATTERN + r"/novels" - test = ("https://www.pixiv.net/en/users/77055466/novels", { - "pattern": "^text:", - "range": "1-5", - "count": 5, - }) + example = "https://www.pixiv.net/en/users/12345/novels" def novels(self): return self.api.user_novels(self.novel_id) @@ -931,10 +713,7 @@ class PixivNovelSeriesExtractor(PixivNovelExtractor): """Extractor for pixiv novel series""" subcategory = "novel-series" pattern = BASE_PATTERN + r"/novel/series/(\d+)" - test = ("https://www.pixiv.net/novel/series/10278364", { - "count": 4, - "content": "b06abed001b3f6ccfb1579699e9a238b46d38ea2", - }) + example = "https://www.pixiv.net/novel/series/12345" def novels(self): return self.api.novel_series(self.novel_id) @@ -945,13 +724,7 @@ class PixivNovelBookmarkExtractor(PixivNovelExtractor): subcategory = "novel-bookmark" pattern = (USER_PATTERN + r"/bookmarks/novels" r"(?:/([^/?#]+))?(?:/?\?([^#]+))?") - test = ( - ("https://www.pixiv.net/en/users/77055466/bookmarks/novels", { - "count": 1, - "content": "7194e8faa876b2b536f185ee271a2b6e46c69089", - }), - ("https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide"), - ) + example = "https://www.pixiv.net/en/users/12345/bookmarks/novels" def __init__(self, match): PixivNovelExtractor.__init__(self, match) @@ -981,11 +754,7 @@ class PixivSketchExtractor(Extractor): root = "https://sketch.pixiv.net" cookies_domain = ".pixiv.net" pattern = r"(?:https?://)?sketch\.pixiv\.net/@([^/?#]+)" - test = ("https://sketch.pixiv.net/@nicoby", { - "pattern": r"https://img\-sketch\.pixiv\.net/uploads/medium" - r"/file/\d+/\d+\.(jpg|png)", - "count": ">= 35", - }) + example = "https://sketch.pixiv.net/@USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/pixnet.py b/gallery_dl/extractor/pixnet.py index 15be5635..eb3edc3f 100644 --- a/gallery_dl/extractor/pixnet.py +++ b/gallery_dl/extractor/pixnet.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2020 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -11,7 +11,6 @@ from .common import Extractor, Message from .. import text, exception - BASE_PATTERN = r"(?:https?://)?(?!www\.)([\w-]+)\.pixnet.net" @@ -68,11 +67,7 @@ class PixnetImageExtractor(PixnetExtractor): filename_fmt = "{id}.{extension}" directory_fmt = ("{category}", "{blog}") pattern = BASE_PATTERN + r"/album/photo/(\d+)" - test = ("https://albertayu773.pixnet.net/album/photo/159443828", { - "url": "156564c422138914c9fa5b42191677b45c414af4", - "keyword": "19971bcd056dfef5593f4328a723a9602be0f087", - "content": "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", - }) + example = "https://USER.pixnet.net/album/photo/12345" def items(self): url = "https://api.pixnet.cc/oembed" @@ -100,19 +95,7 @@ class PixnetSetExtractor(PixnetExtractor): directory_fmt = ("{category}", "{blog}", "{folder_id} {folder_title}", "{set_id} {set_title}") pattern = BASE_PATTERN + r"/album/set/(\d+)" - test = ( - ("https://albertayu773.pixnet.net/album/set/15078995", { - "url": "6535712801af47af51110542f4938a7cef44557f", - "keyword": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", - }), - ("https://anrine910070.pixnet.net/album/set/5917493", { - "url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", - "keyword": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", - }), - ("https://sky92100.pixnet.net/album/set/17492544", { - "count": 0, # password-protected - }), - ) + example = "https://USER.pixnet.net/album/set/12345" def items(self): url = self.url_fmt.format(self.root, self.item_id) @@ -157,10 +140,7 @@ class PixnetFolderExtractor(PixnetExtractor): subcategory = "folder" url_fmt = "{}/album/folder/{}" pattern = BASE_PATTERN + r"/album/folder/(\d+)" - test = ("https://albertayu773.pixnet.net/album/folder/1405768", { - "pattern": PixnetSetExtractor.pattern, - "count": ">= 15", - }) + example = "https://USER.pixnet.net/album/folder/12345" class PixnetUserExtractor(PixnetExtractor): @@ -168,16 +148,4 @@ class PixnetUserExtractor(PixnetExtractor): subcategory = "user" url_fmt = "{}{}/album/list" pattern = BASE_PATTERN + r"()(?:/blog|/album(?:/list)?)?/?(?:$|[?#])" - test = ( - ("https://albertayu773.pixnet.net/"), - ("https://albertayu773.pixnet.net/blog"), - ("https://albertayu773.pixnet.net/album"), - ("https://albertayu773.pixnet.net/album/list", { - "pattern": PixnetFolderExtractor.pattern, - "count": ">= 30", - }), - ("https://anrine910070.pixnet.net/album/list", { - "pattern": PixnetSetExtractor.pattern, - "count": ">= 14", - }), - ) + example = "https://USER.pixnet.net/" diff --git a/gallery_dl/extractor/plurk.py b/gallery_dl/extractor/plurk.py index 41352597..5a3bf5a9 100644 --- a/gallery_dl/extractor/plurk.py +++ b/gallery_dl/extractor/plurk.py @@ -71,10 +71,7 @@ class PlurkTimelineExtractor(PlurkExtractor): """Extractor for URLs from all posts in a Plurk timeline""" subcategory = "timeline" pattern = r"(?:https?://)?(?:www\.)?plurk\.com/(?!p/)(\w+)/?(?:$|[?#])" - test = ("https://www.plurk.com/plurkapi", { - "pattern": r"https?://.+", - "count": ">= 23" - }) + example = "https://www.plurk.com/USER" def __init__(self, match): PlurkExtractor.__init__(self, match) @@ -105,16 +102,7 @@ class PlurkPostExtractor(PlurkExtractor): """Extractor for URLs from a Plurk post""" subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?plurk\.com/p/(\w+)" - test = ( - ("https://www.plurk.com/p/i701j1", { - "url": "2115f208564591b8748525c2807a84596aaaaa5f", - "count": 3, - }), - ("https://www.plurk.com/p/i701j1", { - "options": (("comments", True),), - "count": ">= 210", - }), - ) + example = "https://www.plurk.com/p/12345" def __init__(self, match): PlurkExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/poipiku.py b/gallery_dl/extractor/poipiku.py index e3bb512b..f42016fc 100644 --- a/gallery_dl/extractor/poipiku.py +++ b/gallery_dl/extractor/poipiku.py @@ -96,15 +96,7 @@ class PoipikuUserExtractor(PoipikuExtractor): subcategory = "user" pattern = (BASE_PATTERN + r"/(?:IllustListPcV\.jsp\?PG=(\d+)&ID=)?" r"(\d+)/?(?:$|[?&#])") - test = ( - ("https://poipiku.com/25049/", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049" - r"/\d+_\w+\.(jpe?g|png)$", - "range": "1-10", - "count": 10, - }), - ("https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=") - ) + example = "https://poipiku.com/12345/" def __init__(self, match): PoipikuExtractor.__init__(self, match) @@ -136,54 +128,7 @@ class PoipikuPostExtractor(PoipikuExtractor): """Extractor for a poipiku post""" subcategory = "post" pattern = BASE_PATTERN + r"/(\d+)/(\d+)" - test = ( - ("https://poipiku.com/25049/5864576.html", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049" - r"/005864576_EWN1Y65gQ\.png$", - "keyword": { - "count": "1", - "description": "", - "extension": "png", - "filename": "005864576_EWN1Y65gQ", - "num": 1, - "post_category": "DOODLE", - "post_id": "5864576", - "user_id": "25049", - "user_name": "ユキウサギ", - }, - }), - ("https://poipiku.com/2166245/6411749.html", { - "pattern": r"https://img-org\.poipiku\.com/user_img\d+/002166245" - r"/006411749_\w+\.jpeg$", - "count": 4, - "keyword": { - "count": "4", - "description": "絵茶の産物ネタバレあるやつ", - "num": int, - "post_category": "SPOILER", - "post_id": "6411749", - "user_id": "2166245", - "user_name": "wadahito", - }, - }), - # different warning button style - ("https://poipiku.com/3572553/5776587.html", { - "pattern": r"https://img-org\.poipiku.com/user_img\d+/003572553" - r"/005776587_(\d+_)?\w+\.jpeg$", - "count": 3, - "keyword": { - "count": "3", - "description": "ORANGE OASISボスネタバレ
曲も大好き
" - "2枚目以降はほとんど見えなかった1枚目背景" - "のヒエログリフ小ネタです𓀀", - "num": int, - "post_category": "SPOILER", - "post_id": "5776587", - "user_id": "3572553", - "user_name": "nagakun", - }, - }), - ) + example = "https://poipiku.com/12345/12345.html" def __init__(self, match): PoipikuExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/pornhub.py b/gallery_dl/extractor/pornhub.py index d3619da6..c5ce8327 100644 --- a/gallery_dl/extractor/pornhub.py +++ b/gallery_dl/extractor/pornhub.py @@ -19,6 +19,35 @@ class PornhubExtractor(Extractor): category = "pornhub" root = "https://www.pornhub.com" + def _init(self): + self.cookies.set( + "accessAgeDisclaimerPH", "1", domain=".pornhub.com") + + def _pagination(self, user, path): + if "/" not in path: + path += "/public" + + url = "{}/{}/{}/ajax".format(self.root, user, path) + params = {"page": 1} + headers = { + "Referer": url[:-5], + "X-Requested-With": "XMLHttpRequest", + } + + while True: + response = self.request( + url, method="POST", headers=headers, params=params, + allow_redirects=False) + + if 300 <= response.status_code < 400: + url = "{}{}/{}/ajax".format( + self.root, response.headers["location"], path) + continue + + yield response.text + + params["page"] += 1 + class PornhubGalleryExtractor(PornhubExtractor): """Extractor for image galleries on pornhub.com""" @@ -27,30 +56,7 @@ class PornhubGalleryExtractor(PornhubExtractor): filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/album/(\d+)" - test = ( - ("https://www.pornhub.com/album/19289801", { - "pattern": r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", - "count": ">= 300", - "keyword": { - "id" : int, - "num" : int, - "score" : int, - "views" : int, - "caption": str, - "user" : "Danika Mori", - "gallery": { - "id" : 19289801, - "score": int, - "views": int, - "tags" : list, - "title": "Danika Mori Best Moments", - }, - }, - }), - ("https://www.pornhub.com/album/69040172", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://www.pornhub.com/album/12345" def __init__(self, match): PornhubExtractor.__init__(self, match) @@ -58,9 +64,6 @@ class PornhubGalleryExtractor(PornhubExtractor): self._first = None def items(self): - self.cookies.set( - "accessAgeDisclaimerPH", "1", domain=".pornhub.com") - data = self.metadata() yield Message.Directory, data for num, image in enumerate(self.images(), 1): @@ -116,52 +119,95 @@ class PornhubGalleryExtractor(PornhubExtractor): return +class PornhubGifExtractor(PornhubExtractor): + """Extractor for pornhub.com gifs""" + subcategory = "gif" + directory_fmt = ("{category}", "{user}", "gifs") + filename_fmt = "{id} {title}.{extension}" + archive_fmt = "{id}" + pattern = BASE_PATTERN + r"/gif/(\d+)" + example = "https://www.pornhub.com/gif/12345" + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.gallery_id = match.group(1) + + def items(self): + url = "{}/gif/{}".format(self.root, self.gallery_id) + extr = text.extract_from(self.request(url).text) + + gif = { + "id" : self.gallery_id, + "tags" : extr("data-context-tag='", "'").split(","), + "title": extr('"name": "', '"'), + "url" : extr('"contentUrl": "', '"'), + "date" : text.parse_datetime( + extr('"uploadDate": "', '"'), "%Y-%m-%d"), + "user" : extr('data-mxptext="', '"'), + } + + yield Message.Directory, gif + yield Message.Url, gif["url"], text.nameext_from_url(gif["url"], gif) + + class PornhubUserExtractor(PornhubExtractor): - """Extractor for all galleries of a pornhub user""" + """Extractor for a pornhub user""" subcategory = "user" - pattern = (BASE_PATTERN + r"/(users|model|pornstar)/([^/?#]+)" - "(?:/photos(?:/(public|private|favorites))?)?/?$") - test = ( - ("https://www.pornhub.com/pornstar/danika-mori/photos", { - "pattern": PornhubGalleryExtractor.pattern, - "count": ">= 6", - }), - ("https://www.pornhub.com/users/flyings0l0/"), - ("https://www.pornhub.com/users/flyings0l0/photos/public"), - ("https://www.pornhub.com/users/flyings0l0/photos/private"), - ("https://www.pornhub.com/users/flyings0l0/photos/favorites"), - ("https://www.pornhub.com/model/bossgirl/photos"), - ) + pattern = BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)/?$" + example = "https://www.pornhub.com/model/USER" def __init__(self, match): PornhubExtractor.__init__(self, match) - self.type, self.user, self.cat = match.groups() + self.user = match.group(1) + + def initialize(self): + pass def items(self): - url = "{}/{}/{}/photos/{}/ajax".format( - self.root, self.type, self.user, self.cat or "public") - params = {"page": 1} - headers = { - "Referer": url[:-5], - "X-Requested-With": "XMLHttpRequest", - } + base = "{}/{}/".format(self.root, self.user) + return self._dispatch_extractors(( + (PornhubPhotosExtractor, base + "photos"), + (PornhubGifsExtractor , base + "gifs"), + ), ("photos",)) - data = {"_extractor": PornhubGalleryExtractor} - while True: - response = self.request( - url, method="POST", headers=headers, params=params, - allow_redirects=False) - if 300 <= response.status_code < 400: - url = "{}{}/photos/{}/ajax".format( - self.root, response.headers["location"], - self.cat or "public") - continue +class PornhubPhotosExtractor(PornhubExtractor): + """Extractor for all galleries of a pornhub user""" + subcategory = "photos" + pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" + "/(photos(?:/[^/?#]+)?)") + example = "https://www.pornhub.com/model/USER/photos" + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.user, self.path = match.groups() + def items(self): + data = {"_extractor": PornhubGalleryExtractor} + for page in self._pagination(self.user, self.path): gid = None - for gid in text.extract_iter(response.text, 'id="albumphoto', '"'): + for gid in text.extract_iter(page, 'id="albumphoto', '"'): yield Message.Queue, self.root + "/album/" + gid, data if gid is None: return - params["page"] += 1 + +class PornhubGifsExtractor(PornhubExtractor): + """Extractor for a pornhub user's gifs""" + subcategory = "gifs" + pattern = (BASE_PATTERN + r"/((?:users|model|pornstar)/[^/?#]+)" + "/(gifs(?:/[^/?#]+)?)") + example = "https://www.pornhub.com/model/USER/gifs" + + def __init__(self, match): + PornhubExtractor.__init__(self, match) + self.user, self.path = match.groups() + + def items(self): + data = {"_extractor": PornhubGifExtractor} + for page in self._pagination(self.user, self.path): + gid = None + for gid in text.extract_iter(page, 'id="gif', '"'): + yield Message.Queue, self.root + "/gif/" + gid, data + if gid is None: + return diff --git a/gallery_dl/extractor/pornpics.py b/gallery_dl/extractor/pornpics.py index 929e0f58..4a6f0314 100644 --- a/gallery_dl/extractor/pornpics.py +++ b/gallery_dl/extractor/pornpics.py @@ -24,9 +24,6 @@ class PornpicsExtractor(Extractor): super().__init__(match) self.item = match.group(1) - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): for gallery in self.galleries(): gallery["_extractor"] = PornpicsGalleryExtractor @@ -64,33 +61,7 @@ class PornpicsExtractor(Extractor): class PornpicsGalleryExtractor(PornpicsExtractor, GalleryExtractor): """Extractor for pornpics galleries""" pattern = BASE_PATTERN + r"(/galleries/(?:[^/?#]+-)?(\d+))" - test = ( - (("https://www.pornpics.com/galleries/british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest-62610699/"), { - "pattern": r"https://cdni\.pornpics\.com/1280/7/160/62610699" - r"/62610699_\d+_[0-9a-f]{4}\.jpg", - "keyword": { - "categories": ["MILF", "Amateur", "Sexy", "Outdoor"], - "channel": "FTV MILFs", - "count": 17, - "gallery_id": 62610699, - "models": ["Danielle"], - "num": int, - "slug": "british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest", - "tags": ["Amateur MILF", "Sexy MILF"], - "title": "British beauty Danielle flashes " - "hot breasts, ass and snatch in the forest", - "views": int, - }, - }), - ("https://pornpics.com/es/galleries/62610699", { - "keyword": { - "slug": "british-beauty-danielle-flashes-" - "hot-breasts-ass-and-snatch-in-the-forest", - }, - }), - ) + example = "https://www.pornpics.com/galleries/TITLE-12345/" def __init__(self, match): PornpicsExtractor.__init__(self, match) @@ -126,14 +97,7 @@ class PornpicsTagExtractor(PornpicsExtractor): """Extractor for galleries from pornpics tag searches""" subcategory = "tag" pattern = BASE_PATTERN + r"/tags/([^/?#]+)" - test = ( - ("https://www.pornpics.com/tags/summer-dress/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://pornpics.com/fr/tags/summer-dress"), - ) + example = "https://www.pornpics.com/tags/TAGS/" def galleries(self): url = "{}/tags/{}/".format(self.root, self.item) @@ -144,26 +108,7 @@ class PornpicsSearchExtractor(PornpicsExtractor): """Extractor for galleries from pornpics search results""" subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?q=|pornstars/|channels/)([^/&#]+)" - test = ( - ("https://www.pornpics.com/?q=nature", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://www.pornpics.com/channels/femjoy/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://www.pornpics.com/pornstars/emma-brown/", { - "pattern": PornpicsGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }), - ("https://pornpics.com/jp/?q=nature"), - ("https://pornpics.com/it/channels/femjoy"), - ("https://pornpics.com/pt/pornstars/emma-brown"), - ) + example = "https://www.pornpics.com/?q=QUERY" def galleries(self): url = self.root + "/search/srch.php" diff --git a/gallery_dl/extractor/pururin.py b/gallery_dl/extractor/pururin.py index e56dbcd6..3a4c614b 100644 --- a/gallery_dl/extractor/pururin.py +++ b/gallery_dl/extractor/pururin.py @@ -15,37 +15,9 @@ from .. import text, util class PururinGalleryExtractor(GalleryExtractor): """Extractor for image galleries on pururin.io""" category = "pururin" - pattern = r"(?:https?://)?(?:www\.)?pururin\.[ti]o/(?:gallery|read)/(\d+)" - test = ( - ("https://pururin.to/gallery/38661/iowant-2", { - "pattern": r"https://i\.pururin\.to/38661/\d+\.jpg", - "keyword": { - "title" : "re:I ?owant 2!!", - "title_en" : "re:I ?owant 2!!", - "title_ja" : "", - "gallery_id": 38661, - "count" : 19, - "artist" : ["Shoda Norihiro"], - "group" : ["Obsidian Order"], - "parody" : ["Kantai Collection"], - "characters": ["Iowa", "Teitoku"], - "tags" : list, - "type" : "Doujinshi", - "collection": ["I owant you!"], - "convention": ["C92"], - "rating" : float, - "uploader" : "demo", - "scanlator" : ["mrwayne", "The Lost Light"], - "lang" : "en", - "language" : "English", - } - }), - ("https://pururin.to/gallery/7661/unisis-team-vanilla", { - "count": 17, - }), - ("https://pururin.io/gallery/38661/iowant-2"), - ) root = "https://pururin.to" + pattern = r"(?:https?://)?(?:www\.)?pururin\.[ti]o/(?:gallery|read)/(\d+)" + example = "https://pururin.to/gallery/12345/TITLE" def __init__(self, match): self.gallery_id = match.group(1) diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index ba571bbd..9a6c8a5a 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -34,7 +34,6 @@ class ReactorExtractor(BaseExtractor): self.category = netloc.rpartition(".")[0] def _init(self): - self.session.headers["Referer"] = self.root self.gif = self.config("gif", False) def items(self): @@ -173,25 +172,7 @@ class ReactorTagExtractor(ReactorExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "{search_tags}_{post_id}_{num}" pattern = BASE_PATTERN + r"/tag/([^/?#]+)(?:/[^/?#]+)?" - test = ( - ("http://reactor.cc/tag/gif"), - ("http://anime.reactor.cc/tag/Anime+Art"), - ("http://joyreactor.cc/tag/Advent+Cirno", { - "count": ">= 15", - }), - ("http://joyreactor.com/tag/Cirno", { - "url": "aa59090590b26f4654881301fe8fe748a51625a8", - }), - # 'best' rating (#3073) - ("http://joyreactor.com/tag/Dark+Souls+2/best", { - "count": 4, - }), - ("http://pornreactor.cc/tag/RiceGnat", { - "range": "1-25", - "count": ">= 25", - }), - ("http://fapreactor.com/tag/RiceGnat"), - ) + example = "http://reactor.cc/tag/TAG" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -207,19 +188,7 @@ class ReactorSearchExtractor(ReactorExtractor): directory_fmt = ("{category}", "search", "{search_tags}") archive_fmt = "s_{search_tags}_{post_id}_{num}" pattern = BASE_PATTERN + r"/search(?:/|\?q=)([^/?#]+)" - test = ( - ("http://reactor.cc/search?q=Art"), - ("http://joyreactor.cc/search/Nature", { - "range": "1-25", - "count": ">= 20", - }), - ("http://joyreactor.com/search?q=Nature", { - "range": "1-25", - "count": ">= 20", - }), - ("http://pornreactor.cc/search?q=ecchi+hentai"), - ("http://fapreactor.com/search/ecchi+hentai"), - ) + example = "http://reactor.cc/search?q=QUERY" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -234,19 +203,7 @@ class ReactorUserExtractor(ReactorExtractor): subcategory = "user" directory_fmt = ("{category}", "user", "{user}") pattern = BASE_PATTERN + r"/user/([^/?#]+)" - test = ( - ("http://reactor.cc/user/Dioklet"), - ("http://anime.reactor.cc/user/Shuster"), - ("http://joyreactor.cc/user/hemantic"), - ("http://joyreactor.com/user/Tacoman123", { - "url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", - }), - ("http://pornreactor.cc/user/Disillusion", { - "range": "1-25", - "count": ">= 20", - }), - ("http://fapreactor.com/user/Disillusion"), - ) + example = "http://reactor.cc/user/USER" def __init__(self, match): ReactorExtractor.__init__(self, match) @@ -260,42 +217,7 @@ class ReactorPostExtractor(ReactorExtractor): """Extractor for single posts on *reactor.cc sites""" subcategory = "post" pattern = BASE_PATTERN + r"/post/(\d+)" - test = ( - ("http://reactor.cc/post/4999736", { - "url": "dfc74d150d7267384d8c229c4b82aa210755daa0", - }), - ("http://anime.reactor.cc/post/3576250"), - ("http://joyreactor.com/post/3721876", { # single image - "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" - r"/cartoon-painting-monster-lake-4841316.jpeg", - "count": 1, - "keyword": "2207a7dfed55def2042b6c2554894c8d7fda386e", - }), - ("http://joyreactor.com/post/3713804", { # 4 images - "pattern": r"http://img\d\.joyreactor\.com/pics/post/full" - r"/movie-tv-godzilla-monsters-\d+\.jpeg", - "count": 4, - "keyword": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", - }), - ("http://joyreactor.com/post/3726210", { # gif / video - "url": "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", - "keyword": "8949d9d5fc469dab264752432efbaa499561664a", - }), - ("http://joyreactor.com/post/3668724", { # youtube embed - "url": "bf1666eddcff10c9b58f6be63fa94e4e13074214", - "keyword": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", - }), - ("http://joyreactor.cc/post/1299", { # "malformed" JSON - "url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", - }), - ("http://pornreactor.cc/post/863166", { - "url": "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", - "content": "ec6b0568bfb1803648744077da082d14de844340", - }), - ("http://fapreactor.com/post/863166", { - "url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", - }), - ) + example = "http://reactor.cc/post/12345" def __init__(self, match): ReactorExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/readcomiconline.py b/gallery_dl/extractor/readcomiconline.py index c68068cb..93e41be1 100644 --- a/gallery_dl/extractor/readcomiconline.py +++ b/gallery_dl/extractor/readcomiconline.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -49,11 +49,7 @@ class ReadcomiconlineIssueExtractor(ReadcomiconlineBase, ChapterExtractor): """Extractor for comic-issues from readcomiconline.li""" subcategory = "issue" pattern = BASE_PATTERN + r"(/Comic/[^/?#]+/[^/?#]+\?)([^#]+)" - test = ("https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", { - "pattern": r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", - "keyword": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", - "count": 36, - }) + example = "https://readcomiconline.li/Comic/TITLE/Issue-123?id=12345" def __init__(self, match): ChapterExtractor.__init__(self, match) @@ -98,16 +94,7 @@ class ReadcomiconlineComicExtractor(ReadcomiconlineBase, MangaExtractor): chapterclass = ReadcomiconlineIssueExtractor subcategory = "comic" pattern = BASE_PATTERN + r"(/Comic/[^/?#]+/?)$" - test = ( - ("https://readcomiconline.li/Comic/W-i-t-c-h", { - "url": "74eb8b9504b4084fcc9367b341300b2c52260918", - "keyword": "3986248e4458fa44a201ec073c3684917f48ee0c", - }), - ("https://readcomiconline.to/Comic/Bazooka-Jules", { - "url": "2f66a467a772df4d4592e97a059ddbc3e8991799", - "keyword": "f5ba5246cd787bb750924d9690cb1549199bd516", - }), - ) + example = "https://readcomiconline.li/Comic/TITLE" def chapters(self, page): results = [] diff --git a/gallery_dl/extractor/recursive.py b/gallery_dl/extractor/recursive.py index 4dd9d5c0..41564842 100644 --- a/gallery_dl/extractor/recursive.py +++ b/gallery_dl/extractor/recursive.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2020 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,7 +9,6 @@ """Recursive extractor""" from .common import Extractor, Message -import requests import re @@ -17,34 +16,16 @@ class RecursiveExtractor(Extractor): """Extractor that fetches URLs from a remote or local source""" category = "recursive" pattern = r"r(?:ecursive)?:" - test = ("recursive:https://pastebin.com/raw/FLwrCYsT", { - "url": "eee86d65c346361b818e8f4b2b307d9429f136a2", - }) + example = "recursive:https://pastebin.com/raw/FLwrCYsT" def items(self): - self.session.mount("file://", FileAdapter()) - page = self.request(self.url.partition(":")[2]).text - del self.session.adapters["file://"] + url = self.url.partition(":")[2] - for match in re.finditer(r"https?://[^\s\"']+", page): - yield Message.Queue, match.group(0), {} - - -class FileAdapter(requests.adapters.BaseAdapter): - """Requests adapter for local files""" - - def send(self, request, **kwargs): - response = requests.Response() - try: - response.raw = open(request.url[7:], "rb") - except OSError: - import io - response.raw = io.BytesIO() - response.status_code = requests.codes.bad_request + if url.startswith("file://"): + with open(url[7:]) as fp: + page = fp.read() else: - response.raw.release_conn = response.raw.close - response.status_code = requests.codes.ok - return response + page = self.request(url).text - def close(self): - pass + for match in re.finditer(r"https?://[^\s\"']+", page): + yield Message.Queue, match.group(0), {} diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 05da7f4a..e26b741c 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -65,7 +65,10 @@ class RedditExtractor(Extractor): media = submission url = media["url"] - if url and url.startswith("https://i.redd.it/"): + if url and url.startswith(( + "https://i.redd.it/", + "https://preview.redd.it/", + )): text.nameext_from_url(url, submission) yield Message.Url, url, submission @@ -94,9 +97,13 @@ class RedditExtractor(Extractor): ' href="', '"'): urls.append((url, submission)) for comment in comments: - for url in text.extract_iter( - comment["body_html"] or "", ' href="', '"'): - urls.append((url, comment)) + html = comment["body_html"] or "" + if ' href="' in html: + comment["submission"] = submission + comment["date"] = text.parse_timestamp( + comment["created_utc"]) + for url in text.extract_iter(html, ' href="', '"'): + urls.append((url, comment)) for url, data in urls: if not url or url[0] == "#": @@ -108,7 +115,11 @@ class RedditExtractor(Extractor): if match: extra.append(match.group(1)) elif not match_user(url) and not match_subreddit(url): + if "preview" in data: + data["_fallback"] = self._previews(data) yield Message.Queue, text.unescape(url), data + if "_fallback" in data: + del data["_fallback"] if not extra or depth == max_depth: return @@ -165,22 +176,30 @@ class RedditExtractor(Extractor): submission["_ytdl_extra"] = {"title": submission["title"]} return submission["url"] + def _previews(self, post): + try: + if "reddit_video_preview" in post["preview"]: + video = post["preview"]["reddit_video_preview"] + if "dash_url" in video: + yield "ytdl:" + video["dash_url"] + if "hls_url" in video: + yield "ytdl:" + video["hls_url"] + except Exception as exc: + self.log.debug("%s: %s", exc.__class__.__name__, exc) + + try: + for image in post["preview"]["images"]: + yield image["source"]["url"] + except Exception as exc: + self.log.debug("%s: %s", exc.__class__.__name__, exc) + class RedditSubredditExtractor(RedditExtractor): """Extractor for URLs from subreddits on reddit.com""" subcategory = "subreddit" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com" r"(/r/[^/?#]+(?:/([a-z]+))?)/?(?:\?([^#]*))?(?:$|#)") - test = ( - ("https://www.reddit.com/r/lavaporn/", { - "range": "1-20", - "count": ">= 20", - }), - ("https://www.reddit.com/r/lavaporn/top/?sort=top&t=month"), - ("https://old.reddit.com/r/lavaporn/"), - ("https://np.reddit.com/r/lavaporn/"), - ("https://m.reddit.com/r/lavaporn/"), - ) + example = "https://www.reddit.com/r/SUBREDDIT/" def __init__(self, match): self.subreddit, sub, params = match.groups() @@ -198,13 +217,7 @@ class RedditHomeExtractor(RedditSubredditExtractor): subcategory = "home" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com" r"((?:/([a-z]+))?)/?(?:\?([^#]*))?(?:$|#)") - test = ( - ("https://www.reddit.com/", { - "range": "1-20", - "count": ">= 20", - }), - ("https://old.reddit.com/top/?sort=top&t=month"), - ) + example = "https://www.reddit.com/" class RedditUserExtractor(RedditExtractor): @@ -212,14 +225,7 @@ class RedditUserExtractor(RedditExtractor): subcategory = "user" pattern = (r"(?:https?://)?(?:\w+\.)?reddit\.com/u(?:ser)?/" r"([^/?#]+(?:/([a-z]+))?)/?(?:\?([^#]*))?$") - test = ( - ("https://www.reddit.com/user/username/", { - "count": ">= 2", - }), - ("https://www.reddit.com/user/username/gilded/?sort=top&t=month"), - ("https://old.reddit.com/user/username/"), - ("https://www.reddit.com/u/username/"), - ) + example = "https://www.reddit.com/user/USER/" def __init__(self, match): self.user, sub, params = match.groups() @@ -238,71 +244,7 @@ class RedditSubmissionExtractor(RedditExtractor): pattern = (r"(?:https?://)?(?:" r"(?:\w+\.)?reddit\.com/(?:(?:r|u|user)/[^/?#]+" r"/comments|gallery)|redd\.it)/([a-z0-9]+)") - test = ( - ("https://www.reddit.com/r/lavaporn/comments/8cqhub/", { - "pattern": r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", - "count": 1, - }), - ("https://www.reddit.com/r/lavaporn/comments/8cqhub/", { - "options": (("comments", 500),), - "pattern": r"https://", - "count": 3, - }), - ("https://www.reddit.com/gallery/hrrh23", { - "url": "25b91ede15459470274dd17291424b037ed8b0ae", - "content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", - "count": 3, - }), - # video - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11", - "count": 1, - }), - # video (ytdl) - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "options": (("videos", "ytdl"),), - "pattern": r"ytdl:https://www.reddit.com/r/aww/comments/90bu6w" - r"/heat_index_was_110_degrees_so_we_offered_him_a/", - "count": 1, - }), - # video (dash) - ("https://www.reddit.com/r/aww/comments/90bu6w/", { - "options": (("videos", "dash"),), - "pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11" - r"/DASHPlaylist.mpd\?a=", - "count": 1, - }), - # deleted gallery (#953) - ("https://www.reddit.com/gallery/icfgzv", { - "count": 0, - }), - # animated gallery items (#955) - ("https://www.reddit.com/r/araragi/comments/ib32hm", { - "pattern": r"https://i\.redd\.it/\w+\.gif", - "count": 2, - }), - # "failed" gallery item (#1127) - ("https://www.reddit.com/r/cosplay/comments/jvwaqr", { - "count": 1, - }), - # gallery with no 'media_metadata' (#2001) - ("https://www.reddit.com/r/kpopfap/comments/qjj04q/", { - "count": 0, - }), - # user page submission (#2301) - ("https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", { - "pattern": r"https://i.redd.it/8fpgv17yqlh81.jpg", - "count": 1, - }), - # cross-posted video (#887, #3586, #3976) - ("https://www.reddit.com/r/kittengifs/comments/12m0b8d", { - "pattern": r"ytdl:https://v\.redd\.it/cvabpjacrvta1", - }), - ("https://old.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://np.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://m.reddit.com/r/lavaporn/comments/2a00np/"), - ("https://redd.it/2a00np/"), - ) + example = "https://www.reddit.com/r/SUBREDDIT/comments/id/" def __init__(self, match): RedditExtractor.__init__(self, match) @@ -319,22 +261,7 @@ class RedditImageExtractor(Extractor): archive_fmt = "{filename}" pattern = (r"(?:https?://)?((?:i|preview)\.redd\.it|i\.reddituploads\.com)" r"/([^/?#]+)(\?[^#]*)?") - test = ( - ("https://i.redd.it/upjtjcx2npzz.jpg", { - "url": "0de614900feef103e580b632190458c0b62b641a", - "content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", - }), - (("https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d" - "?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2"), { - "url": "f24f25efcedaddeec802e46c60d77ef975dc52a5", - "content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", - }), - # preview.redd.it -> i.redd.it - (("https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart" - "&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac"), { - "pattern": r"^https://i\.redd\.it/00af44lpn0u51\.jpg$" - }), - ) + example = "https://i.redd.it/NAME.EXT" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index abd21b30..57093091 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -90,16 +90,7 @@ class RedgifsUserExtractor(RedgifsExtractor): subcategory = "user" directory_fmt = ("{category}", "{userName}") pattern = r"(?:https?://)?(?:\w+\.)?redgifs\.com/users/([^/?#]+)/?$" - test = ( - ("https://www.redgifs.com/users/Natalifiction", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "count": ">= 100", - }), - ("https://v3.redgifs.com/users/lamsinka89", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "count": ">= 100", - }), - ) + example = "https://www.redgifs.com/users/USER" def metadata(self): return {"userName": self.key} @@ -111,31 +102,21 @@ class RedgifsUserExtractor(RedgifsExtractor): class RedgifsCollectionExtractor(RedgifsExtractor): """Extractor for an individual user collection""" subcategory = "collection" - directory_fmt = ("{category}", "{userName}", "{folderName}") - archive_fmt = "{folderId}_{id}" + directory_fmt = ( + "{category}", "{collection[userName]}", "{collection[folderName]}") + archive_fmt = "{collection[folderId]}_{id}" pattern = (r"(?:https?://)?(?:www\.)?redgifs\.com/users" r"/([^/?#]+)/collections/([^/?#]+)") - test = ( - ("https://www.redgifs.com/users/boombah123/collections/2631326bbd", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ("https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ) + example = "https://www.redgifs.com/users/USER/collections/ID" def __init__(self, match): RedgifsExtractor.__init__(self, match) self.collection_id = match.group(2) def metadata(self): - data = {"userName": self.key} - data.update(self.api.collection_info(self.key, self.collection_id)) - return data + collection = self.api.collection_info(self.key, self.collection_id) + collection["userName"] = self.key + return {"collection": collection} def gifs(self): return self.api.collection(self.key, self.collection_id) @@ -146,11 +127,7 @@ class RedgifsCollectionsExtractor(RedgifsExtractor): subcategory = "collections" pattern = (r"(?:https?://)?(?:www\.)?redgifs\.com/users" r"/([^/?#]+)/collections/?$") - test = ("https://www.redgifs.com/users/boombah123/collections", { - "pattern": (r"https://www\.redgifs\.com/users" - r"/boombah123/collections/\w+"), - "count": ">= 3", - }) + example = "https://www.redgifs.com/users/USER/collections" def items(self): for collection in self.api.collections(self.key): @@ -164,18 +141,7 @@ class RedgifsNichesExtractor(RedgifsExtractor): """Extractor for redgifs niches""" subcategory = "niches" pattern = r"(?:https?://)?(?:www\.)?redgifs\.com/niches/([^/?#]+)" - test = ( - ("https://www.redgifs.com/niches/boobs", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ("https://www.redgifs.com/niches/ass", { - "pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "range": "1-20", - "count": 20, - }), - ) + example = "https://www.redgifs.com/niches/NAME" def gifs(self): return self.api.niches(self.key) @@ -187,21 +153,7 @@ class RedgifsSearchExtractor(RedgifsExtractor): directory_fmt = ("{category}", "Search", "{search}") pattern = (r"(?:https?://)?(?:\w+\.)?redgifs\.com" r"/(?:gifs/([^/?#]+)|browse)(?:/?\?([^#]+))?") - test = ( - ("https://www.redgifs.com/gifs/jav", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "range": "1-10", - "count": 10, - }), - ("https://www.redgifs.com/browse?tags=JAV", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "range": "1-10", - "count": 10, - }), - ("https://www.redgifs.com/gifs/jav?order=best&verified=1"), - ("https://www.redgifs.com/browse?type=i&verified=y&order=top7"), - ("https://v3.redgifs.com/browse?tags=JAV"), - ) + example = "https://www.redgifs.com/gifs/TAG" def __init__(self, match): RedgifsExtractor.__init__(self, match) @@ -227,27 +179,7 @@ class RedgifsImageExtractor(RedgifsExtractor): r"(?:\w+\.)?redgifs\.com/(?:watch|ifr)|" r"(?:www\.)?gifdeliverynetwork\.com|" r"i\.redgifs\.com/i)/([A-Za-z]+)") - test = ( - ("https://redgifs.com/watch/foolishforkedabyssiniancat", { - "pattern": r"https://\w+\.redgifs\.com" - r"/FoolishForkedAbyssiniancat\.mp4", - "content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", - }), - # gallery (#4021) - ("https://www.redgifs.com/watch/desertedbaregraywolf", { - "pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", - "count": 4, - "keyword": { - "num": int, - "count": 4, - "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", - }, - }), - ("https://redgifs.com/ifr/FoolishForkedAbyssiniancat"), - ("https://i.redgifs.com/i/FoolishForkedAbyssiniancat"), - ("https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat"), - ("https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat"), - ) + example = "https://redgifs.com/watch/ID" def gifs(self): return (self.api.gif(self.key),) @@ -261,7 +193,6 @@ class RedgifsAPI(): def __init__(self, extractor): self.extractor = extractor self.headers = { - "Referer" : extractor.root + "/", "authorization" : None, "content-type" : "application/json", "x-customheader": extractor.root + "/", diff --git a/gallery_dl/extractor/rule34us.py b/gallery_dl/extractor/rule34us.py index 88331eaa..6439a225 100644 --- a/gallery_dl/extractor/rule34us.py +++ b/gallery_dl/extractor/rule34us.py @@ -10,8 +10,8 @@ from .booru import BooruExtractor from .. import text -import re import collections +import re class Rule34usExtractor(BooruExtractor): @@ -54,11 +54,7 @@ class Rule34usTagExtractor(Rule34usExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = r"(?:https?://)?rule34\.us/index\.php\?r=posts/index&q=([^&#]+)" - test = ("https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", { - "pattern": r"https://img\d*\.rule34\.us" - r"/images/../../[0-9a-f]{32}\.\w+", - "count": 10, - }) + example = "https://rule34.us/index.php?r=posts/index&q=TAG" def __init__(self, match): Rule34usExtractor.__init__(self, match) @@ -95,31 +91,7 @@ class Rule34usPostExtractor(Rule34usExtractor): subcategory = "post" archive_fmt = "{id}" pattern = r"(?:https?://)?rule34\.us/index\.php\?r=posts/view&id=(\d+)" - test = ( - ("https://rule34.us/index.php?r=posts/view&id=3709005", { - "pattern": r"https://img\d*\.rule34\.us/images/14/7b" - r"/147bee6fc2e13f73f5f9bac9d4930b13\.png", - "content": "d714342ea84050f82dda5f0c194d677337abafc5", - }), - ("https://rule34.us/index.php?r=posts/view&id=4576310", { - "pattern": r"https://video\.rule34\.us/images/a2/94" - r"/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", - "keyword": { - "extension": "mp4", - "file_url": str, - "filename": "a294ff8e1f8e0efa041e5dc9d1480011", - "height": "3982", - "id": "4576310", - "md5": "a294ff8e1f8e0efa041e5dc9d1480011", - "score": r"re:\d+", - "tags": "tagme, video", - "tags_general": "video", - "tags_metadata": "tagme", - "uploader": "Anonymous", - "width": "3184", - }, - }), - ) + example = "https://rule34.us/index.php?r=posts/view&id=12345" def __init__(self, match): Rule34usExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index 303a8e55..745a351b 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -88,32 +88,7 @@ class SankakuTagExtractor(SankakuExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" pattern = BASE_PATTERN + r"/?\?([^#]*)" - test = ( - ("https://sankaku.app/?tags=bonocho", { - "count": 5, - "pattern": r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" - r"/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", - }), - ("https://beta.sankakucomplex.com/?tags=bonocho"), - ("https://chan.sankakucomplex.com/?tags=bonocho"), - ("https://black.sankakucomplex.com/?tags=bonocho"), - ("https://white.sankakucomplex.com/?tags=bonocho"), - ("https://sankaku.app/ja?tags=order%3Apopularity"), - ("https://sankaku.app/no/?tags=order%3Apopularity"), - # error on five or more tags - ("https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", { - "options": (("username", None),), - "exception": exception.StopExtraction, - }), - # match arbitrary query parameters - ("https://chan.sankakucomplex.com" - "/?tags=marie_rose&page=98&next=3874906&commit=Search"), - # 'date:' tags (#1790) - ("https://chan.sankakucomplex.com/?tags=date:2023-03-20", { - "range": "1", - "count": 1, - }), - ) + example = "https://sankaku.app/?tags=TAG" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -143,13 +118,7 @@ class SankakuPoolExtractor(SankakuExtractor): directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name_en]}") archive_fmt = "p_{pool}_{id}" pattern = BASE_PATTERN + r"/(?:books|pool/show)/(\d+)" - test = ( - ("https://sankaku.app/books/90", { - "count": 5, - }), - ("https://beta.sankakucomplex.com/books/90"), - ("https://chan.sankakucomplex.com/pool/show/90"), - ) + example = "https://sankaku.app/books/12345" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -175,50 +144,7 @@ class SankakuPostExtractor(SankakuExtractor): subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post/show/([0-9a-f]+)" - test = ( - ("https://sankaku.app/post/show/360451", { - "content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - "options": (("tags", True),), - "keyword": { - "tags_artist" : ["bonocho"], - "tags_studio" : ["dc_comics"], - "tags_medium" : list, - "tags_copyright": list, - "tags_character": list, - "tags_general" : list, - }, - }), - # 'contentious_content' - ("https://sankaku.app/post/show/21418978", { - "pattern": r"https://s\.sankakucomplex\.com" - r"/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", - }), - # empty tags (#1617) - ("https://sankaku.app/post/show/20758561", { - "options": (("tags", True),), - "count": 1, - "keyword": { - "tags": list, - "tags_general": ["key(mangaka)", "key(mangaka)"], - }, - }), - # md5 hexdigest instead of ID (#3952) - (("https://chan.sankakucomplex.com/post/show" - "/f8ba89043078f0e4be2d9c46550b840a"), { - "pattern": r"https://s\.sankakucomplex\.com" - r"/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "count": 1, - "keyword": { - "id": 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", - }, - }), - ("https://chan.sankakucomplex.com/post/show/360451"), - ("https://chan.sankakucomplex.com/ja/post/show/360451"), - ("https://beta.sankakucomplex.com/post/show/360451"), - ("https://white.sankakucomplex.com/post/show/360451"), - ("https://black.sankakucomplex.com/post/show/360451"), - ) + example = "https://sankaku.app/post/show/12345" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -232,13 +158,7 @@ class SankakuBooksExtractor(SankakuExtractor): """Extractor for books by tag search on sankaku.app""" subcategory = "books" pattern = BASE_PATTERN + r"/books/?\?([^#]*)" - test = ( - ("https://sankaku.app/books?tags=aiue_oka", { - "range": "1-20", - "count": 20, - }), - ("https://beta.sankakucomplex.com/books?tags=aiue_oka"), - ) + example = "https://sankaku.app/books?tags=TAG" def __init__(self, match): SankakuExtractor.__init__(self, match) @@ -260,7 +180,6 @@ class SankakuAPI(): self.extractor = extractor self.headers = { "Accept" : "application/vnd.sankaku.api+json;v=2", - "Referer" : extractor.root + "/", "Platform": "web-app", "Origin" : extractor.root, } diff --git a/gallery_dl/extractor/sankakucomplex.py b/gallery_dl/extractor/sankakucomplex.py index 47e067b2..e1d4153a 100644 --- a/gallery_dl/extractor/sankakucomplex.py +++ b/gallery_dl/extractor/sankakucomplex.py @@ -30,32 +30,8 @@ class SankakucomplexArticleExtractor(SankakucomplexExtractor): filename_fmt = "{filename}.{extension}" archive_fmt = "{date:%Y%m%d}_{filename}" pattern = (r"(?:https?://)?www\.sankakucomplex\.com" - r"/(\d{4}/\d\d/\d\d/[^/?#]+)") - test = ( - ("https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", { - "url": "4a9ecc5ae917fbce469280da5b6a482510cae84d", - "keyword": "bfe08310e7d9a572f568f6900e0ed0eb295aa2b3", - }), - ("https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", { - "url": "a1e249173fd6c899a8134fcfbd9c925588a63f7c", - "keyword": "e78fcc23c2711befc0969a45ea5082a29efccf68", - }), - # videos (#308) - (("https://www.sankakucomplex.com/2019/06/11" - "/darling-ol-goddess-shows-off-her-plump-lower-area/"), { - "pattern": r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", - "range": "26-", - "count": 5, - }), - # youtube embeds (#308) - (("https://www.sankakucomplex.com/2015/02/12" - "/snow-miku-2015-live-magical-indeed/"), { - "options": (("embeds", True),), - "pattern": r"https://www.youtube.com/embed/", - "range": "2-", - "count": 2, - }), - ) + r"/(\d\d\d\d/\d\d/\d\d/[^/?#]+)") + example = "https://www.sankakucomplex.com/1970/01/01/TITLE" def items(self): url = "{}/{}/?pg=X".format(self.root, self.path) @@ -113,15 +89,7 @@ class SankakucomplexTagExtractor(SankakucomplexExtractor): subcategory = "tag" pattern = (r"(?:https?://)?www\.sankakucomplex\.com" r"/((?:tag|category|author)/[^/?#]+)") - test = ( - ("https://www.sankakucomplex.com/tag/cosplay/", { - "range": "1-50", - "count": 50, - "pattern": SankakucomplexArticleExtractor.pattern, - }), - ("https://www.sankakucomplex.com/category/anime/"), - ("https://www.sankakucomplex.com/author/rift/page/5/"), - ) + example = "https://www.sankakucomplex.com/tag/TAG/" def items(self): pnum = 1 diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index ab466149..edfe1dcd 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -58,32 +58,7 @@ class SeigaUserExtractor(SeigaExtractor): filename_fmt = "{category}_{user[id]}_{image_id}.{extension}" pattern = (r"(?:https?://)?(?:www\.|(?:sp\.)?seiga\.)?nicovideo\.jp/" r"user/illust/(\d+)(?:\?(?:[^&]+&)*sort=([^&#]+))?") - test = ( - ("https://seiga.nicovideo.jp/user/illust/39537793", { - "pattern": r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", - "count": ">= 4", - "keyword": { - "user": { - "id": 39537793, - "message": str, - "name": str, - }, - "clips": int, - "comments": int, - "count": int, - "extension": None, - "image_id": int, - "title": str, - "views": int, - }, - }), - ("https://seiga.nicovideo.jp/user/illust/79433", { - "exception": exception.NotFoundError, - }), - ("https://seiga.nicovideo.jp/user/illust/39537793" - "?sort=image_view&target=illust_all"), - ("https://sp.seiga.nicovideo.jp/user/illust/39537793"), - ) + example = "https://seiga.nicovideo.jp/user/illust/12345" def __init__(self, match): SeigaExtractor.__init__(self, match) @@ -156,26 +131,7 @@ class SeigaImageExtractor(SeigaExtractor): r"(?:seiga\.|www\.)?nicovideo\.jp/(?:seiga/im|image/source/)" r"|sp\.seiga\.nicovideo\.jp/seiga/#!/im" r"|lohas\.nicoseiga\.jp/(?:thumb|(?:priv|o)/[^/]+/\d+)/)(\d+)") - test = ( - ("https://seiga.nicovideo.jp/seiga/im5977527", { - "keyword": "c8339781da260f7fc44894ad9ada016f53e3b12a", - "content": "d9202292012178374d57fb0126f6124387265297", - }), - ("https://seiga.nicovideo.jp/seiga/im123", { - "exception": exception.NotFoundError, - }), - ("https://seiga.nicovideo.jp/seiga/im10877923", { - "pattern": r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e46" - r"5e0411c0822e0b510e286/1688757110/10877923", - }), - ("https://seiga.nicovideo.jp/image/source/5977527"), - ("https://sp.seiga.nicovideo.jp/seiga/#!/im5977527"), - ("https://lohas.nicoseiga.jp/thumb/5977527i"), - ("https://lohas.nicoseiga.jp/priv" - "/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527"), - ("https://lohas.nicoseiga.jp/o" - "/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527"), - ) + example = "https://seiga.nicovideo.jp/seiga/im12345" def __init__(self, match): SeigaExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/senmanga.py b/gallery_dl/extractor/senmanga.py index c48be84e..a8fdc4c9 100644 --- a/gallery_dl/extractor/senmanga.py +++ b/gallery_dl/extractor/senmanga.py @@ -17,54 +17,7 @@ class SenmangaChapterExtractor(ChapterExtractor): category = "senmanga" root = "https://raw.senmanga.com" pattern = r"(?:https?://)?raw\.senmanga\.com(/[^/?#]+/[^/?#]+)" - test = ( - ("https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", { - "pattern": r"https://raw\.senmanga\.com/viewer" - r"/Bokura-wa-Minna-Kawaisou/37A/[12]", - "url": "5f95140ff511d8497e2ec08fa7267c6bb231faec", - "content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", - "keyword": { - "chapter": "37A", - "count": 2, - "extension": "", - "filename": "re:[12]", - "lang": "ja", - "language": "Japanese", - "manga": "Bokura wa Minna Kawaisou", - "page": int, - }, - }), - ("http://raw.senmanga.com/Love-Lab/2016-03/1", { - "pattern": r"https://raw\.senmanga\.com/viewer" - r"/Love-Lab/2016-03/\d", - "url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", - "keyword": { - "chapter": "2016-03", - "count": 9, - "extension": "", - "filename": r"re:\d", - "manga": "Renai Lab 恋愛ラボ", - }, - }), - ("https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", { - "pattern": r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a" - r"/akabane-honeko-no-bodyguard/chapter-1" - r"/\d+-[0-9a-f]{13}\.jpg", - "keyword": { - "chapter": "1", - "count": 65, - "extension": "jpg", - "filename": r"re:\d+-\w+", - "manga": "Akabane Honeko no Bodyguard", - }, - }), - # no http scheme () - ("https://raw.senmanga.com/amama-cinderella/3", { - "pattern": r"^https://kumacdn.club/image-new-2/a/amama-cinderella" - r"/chapter-3/.+\.jpg", - "count": 30, - }), - ) + example = "https://raw.senmanga.com/MANGA/CHAPTER" def _init(self): self.session.headers["Referer"] = self.gallery_url diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py index c6588def..80f2aea5 100644 --- a/gallery_dl/extractor/sexcom.py +++ b/gallery_dl/extractor/sexcom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -106,43 +106,7 @@ class SexcomPinExtractor(SexcomExtractor): subcategory = "pin" directory_fmt = ("{category}",) pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+)(?!.*#related$)" - test = ( - # picture - ("https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", { - "pattern": "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", - "content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", - "keyword": { - "comments" : int, - "date" : "dt:2014-10-19 15:45:44", - "extension": "jpg", - "filename" : "7637609", - "likes" : int, - "pin_id" : 21241874, - "repins" : int, - "tags" : list, - "thumbnail": str, - "title" : "Sexy Ecchi Girls 166", - "type" : "picture", - "uploader" : "mangazeta", - "url" : str, - }, - }), - # gif - ("https://www.sex.com/pin/55435122-ecchi/", { - "pattern": "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", - "content": "176cc63fa05182cb0438c648230c0f324a5965fe", - }), - # video - ("https://www.sex.com/pin/55748341/", { - "pattern": r"https://cdn\.sex\.com/videos/pinporn" - r"/2018/02/10/776229_hd\.mp4", - "content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", - }), - # pornhub embed - ("https://www.sex.com/pin/55847384-very-nicely-animated/", { - "pattern": "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", - }), - ) + example = "https://www.sex.com/pin/12345-TITLE/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -157,9 +121,7 @@ class SexcomRelatedPinExtractor(SexcomPinExtractor): subcategory = "related-pin" directory_fmt = ("{category}", "related {original_pin[pin_id]}") pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+).*#related$" - test = ("https://www.sex.com/pin/21241874/#related", { - "count": ">= 20", - }) + example = "https://www.sex.com/pin/12345#related" def metadata(self): pin = self._parse_pin(SexcomPinExtractor.pins(self)[0]) @@ -176,9 +138,7 @@ class SexcomPinsExtractor(SexcomExtractor): subcategory = "pins" directory_fmt = ("{category}", "{user}") pattern = r"(?:https?://)?(?:www\.)?sex\.com/user/([^/?#]+)/pins/" - test = ("https://www.sex.com/user/sirjuan79/pins/", { - "count": ">= 15", - }) + example = "https://www.sex.com/user/USER/pins/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -198,9 +158,7 @@ class SexcomBoardExtractor(SexcomExtractor): directory_fmt = ("{category}", "{user}", "{board}") pattern = (r"(?:https?://)?(?:www\.)?sex\.com/user" r"/([^/?#]+)/(?!(?:following|pins|repins|likes)/)([^/?#]+)") - test = ("https://www.sex.com/user/ronin17/exciting-hentai/", { - "count": ">= 15", - }) + example = "https://www.sex.com/user/USER/BOARD/" def __init__(self, match): SexcomExtractor.__init__(self, match) @@ -224,17 +182,7 @@ class SexcomSearchExtractor(SexcomExtractor): pattern = (r"(?:https?://)?(?:www\.)?sex\.com/((?:" r"(pic|gif|video)s/([^/?#]*)|search/(pic|gif|video)s" r")/?(?:\?([^#]+))?)") - test = ( - ("https://www.sex.com/search/pics?query=ecchi", { - "range": "1-10", - "count": 10, - }), - ("https://www.sex.com/videos/hentai/", { - "range": "1-10", - "count": 10, - }), - ("https://www.sex.com/pics/?sort=popular&sub=all&page=1"), - ) + example = "https://www.sex.com/search/pics?query=QUERY" def __init__(self, match): SexcomExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/shimmie2.py b/gallery_dl/extractor/shimmie2.py index 4ae4e96e..99dc9034 100644 --- a/gallery_dl/extractor/shimmie2.py +++ b/gallery_dl/extractor/shimmie2.py @@ -100,36 +100,7 @@ class Shimmie2TagExtractor(Shimmie2Extractor): directory_fmt = ("{category}", "{search_tags}") file_url_fmt = "{}/_images/{}/{}%20-%20{}.{}" pattern = BASE_PATTERN + r"post/list/([^/?#]+)(?:/(\d+))?()" - test = ( - ("https://meme.museum/post/list/animated/1", { - "pattern": r"https://meme\.museum/_images/\w+/\d+%20-%20", - "count": ">= 30" - }), - ("https://loudbooru.com/post/list/original_character/1", { - "pattern": r"https://loudbooru\.com/_images/[0-9a-f]{32}/\d+", - "range": "1-100", - "count": 100, - }), - ("https://giantessbooru.com/index.php?q=/post/list/drawing/1", { - "pattern": r"https://giantessbooru\.com/index\.php" - r"\?q=/image/\d+\.jpg", - "range": "1-100", - "count": 100, - }), - ("https://giantessbooru.com/post/list/drawing/1"), - ("https://tentaclerape.net/post/list/comic/1", { - "pattern": r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", - "range": "1-100", - "count": 100, - }), - ("https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", { - "pattern": r"https://booru\.cavemanon\.xyz" - r"/index\.php\?q=image/\d+\.\w+", - "range": "1-100", - "count": 100, - }), - ("https://booru.cavemanon.xyz/post/list/Amber/1"), - ) + example = "https://loudbooru.com/post/list/TAG/1" def __init__(self, match): Shimmie2Extractor.__init__(self, match) @@ -225,115 +196,7 @@ class Shimmie2PostExtractor(Shimmie2Extractor): """Extractor for single shimmie2 posts""" subcategory = "post" pattern = BASE_PATTERN + r"post/view/(\d+)" - test = ( - ("https://meme.museum/post/view/10243", { - "pattern": r"https://meme\.museum/_images/105febebcd5ca791ee332adc" - r"49971f78/10243%20-%20g%20beard%20open_source%20richar" - r"d_stallman%20stallman%20tagme%20text\.jpg", - "content": "45565f3f141fc960a8ae1168b80e718a494c52d2", - "keyword": { - "extension": "jpg", - "file_url": "https://meme.museum/_images/105febebcd5ca791ee332" - "adc49971f78/10243%20-%20g%20beard%20open_source%2" - "0richard_stallman%20stallman%20tagme%20text.jpg", - "filename": "10243 - g beard open_source richard_stallman " - "stallman tagme text", - "height": 451, - "id": 10243, - "md5": "105febebcd5ca791ee332adc49971f78", - "size": 0, - "subcategory": "post", - "tags": "/g/ beard open_source " - "richard_stallman stallman tagme text", - "width": 480, - }, - }), - ("https://loudbooru.com/post/view/33828", { - "pattern": r"https://loudbooru\.com/_images/.+\.png", - "content": "a4755f787ba23ae2aa297a46810f802ca9032739", - "keyword": { - "extension": "png", - "file_url": "https://loudbooru.com/_images/ca2638d903c86e8337f" - "e9aeb4974be88/33828%20-%202020%20artist%3Astikyfi" - "nkaz%20character%3Alisa_loud%20cover%20fanfiction" - "%3Aplatz_eins%20frowning%20half-closed_eyes%20sol" - "o%20text%20title_card.png", - "filename": "33828 - 2020 artist:stikyfinkaz character:lisa_" - "loud cover fanfiction:platz_eins frowning " - "half-closed_eyes solo text title_card", - "height": 1920, - "id": 33828, - "md5": "ca2638d903c86e8337fe9aeb4974be88", - "tags": "2020 artist:stikyfinkaz character:lisa_loud cover " - "fanfiction:platz_eins frowning half-closed_eyes " - "solo text title_card", - "width": 1078, - }, - }), - ("https://giantessbooru.com/index.php?q=/post/view/41", { - "pattern": r"https://giantessbooru\.com/index\.php" - r"\?q=/image/41\.jpg", - "content": "79115ed309d1f4e82e7bead6948760e889139c91", - "keyword": { - "extension": "jpg", - "file_url": "https://giantessbooru.com/index.php" - "?q=/image/41.jpg", - "filename": "41", - "height": 0, - "id": 41, - "md5": "", - "size": 0, - "tags": "anime bare_midriff color drawing gentle giantess " - "karbo looking_at_tinies negeyari outdoors smiling " - "snake_girl white_hair", - "width": 1387, - }, - }), - ("https://giantessbooru.com/post/view/41"), - ("https://tentaclerape.net/post/view/10", { - "pattern": r"https://tentaclerape\.net/\./index\.php" - r"\?q=/image/10\.jpg", - "content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", - "keyword": { - "extension": "jpg", - "file_url": "https://tentaclerape.net/./index.php" - "?q=/image/10.jpg", - "filename": "10", - "height": 427, - "id": 10, - "md5": "945db71eeccaef82ce44b77564260c0b", - "size": 0, - "subcategory": "post", - "tags": "Deviant_Art Pet Tentacle artist_sche blonde_hair " - "blouse boots green_eyes highheels leash miniskirt " - "octopus schoolgirl white_skin willing", - "width": 300, - }, - }), - # video - ("https://tentaclerape.net/post/view/91267", { - "pattern": r"https://tentaclerape\.net/\./index\.php" - r"\?q=/image/91267\.mp4", - }), - ("https://booru.cavemanon.xyz/index.php?q=post/view/8335", { - "pattern": r"https://booru\.cavemanon\.xyz" - r"/index\.php\?q=image/8335\.png", - "content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", - "keyword": { - "extension": "png", - "file_url": "https://booru.cavemanon.xyz" - "/index.php?q=image/8335.png", - "filename": "8335", - "height": 460, - "id": 8335, - "md5": "", - "size": 0, - "tags": "Color Fang Food Pterodactyl " - "discord_emote transparent", - "width": 459, - }, - }), - ) + example = "https://loudbooru.com/post/view/12345" def __init__(self, match): Shimmie2Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/shopify.py b/gallery_dl/extractor/shopify.py index f6e8bc03..a658cac4 100644 --- a/gallery_dl/extractor/shopify.py +++ b/gallery_dl/extractor/shopify.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -95,24 +95,7 @@ class ShopifyCollectionExtractor(ShopifyExtractor): subcategory = "collection" directory_fmt = ("{category}", "{collection[title]}") pattern = BASE_PATTERN + r"(/collections/[\w-]+)/?(?:$|[?#])" - test = ( - ("https://chelseacrew.com/collections/flats"), - ("https://www.fashionnova.com/collections/mini-dresses", { - "range": "1-20", - "count": 20, - }), - ("https://www.fashionnova.com/collections/mini-dresses/?page=1"), - ("https://www.fashionnova.com/collections/mini-dresses#1"), - ("https://loungeunderwear.com/collections/apparel"), - ("https://michaels.com.au/collections/microphones"), - ("https://modcloth.com/collections/shoes"), - ("https://www.ohpolly.com/collections/dresses-mini-dresses"), - ("https://www.omgmiamiswimwear.com/collections/fajas"), - ("https://pinupgirlclothing.com/collections/evening"), - ("https://www.raidlondon.com/collections/flats"), - ("https://www.unique-vintage.com/collections/flapper-1920s"), - ("https://www.windsorstore.com/collections/dresses-ball-gowns"), - ) + example = "https://www.fashionnova.com/collections/TITLE" def metadata(self): return self.request(self.item_url + ".json").json() @@ -134,32 +117,7 @@ class ShopifyProductExtractor(ShopifyExtractor): subcategory = "product" directory_fmt = ("{category}", "Products") pattern = BASE_PATTERN + r"((?:/collections/[\w-]+)?/products/[\w-]+)" - test = ( - ("https://chelseacrew.com/collections/flats/products/dora"), - ("https://www.fashionnova.com/products/essential-slide-red", { - "pattern": r"https?://cdn\d*\.shopify.com/", - "count": 3, - }), - ("https://www.fashionnova.com/collections/flats/products/name"), - ("https://de.loungeunderwear.com/products/ribbed-crop-top-black"), - ("https://michaels.com.au/collections/audio/products" - "/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281"), - ("https://modcloth.com/collections/shoes/products/heidii-brn"), - (("https://www.ohpolly.com/products/edonia-ruched-triangle-cup" - "-a-line-mini-dress-brown")), - ("https://www.omgmiamiswimwear.com/products/la-medusa-maxi-dress", { - "pattern": r"https://cdn\.shopify\.com/s/files/1/1819/6171/", - "count": 5, - }), - ("https://pinupgirlclothing.com/collections/evening/products" - "/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design"), - ("https://www.raidlondon.com/collections/flats/products" - "/raid-addyson-chunky-flat-shoe-in-white"), - ("https://www.unique-vintage.com/collections/flapper-1920s/products" - "/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress"), - ("https://www.windsorstore.com/collections/accessories-belts/products" - "/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001"), - ) + example = "https://www.fashionnova.com/collections/TITLE/products/NAME" def products(self): product = self.request(self.item_url + ".json").json()["product"] diff --git a/gallery_dl/extractor/simplyhentai.py b/gallery_dl/extractor/simplyhentai.py index d1ccc492..6f722916 100644 --- a/gallery_dl/extractor/simplyhentai.py +++ b/gallery_dl/extractor/simplyhentai.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2021 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,20 +19,7 @@ class SimplyhentaiGalleryExtractor(GalleryExtractor): pattern = (r"(?:https?://)?(?!videos\.)([\w-]+\.)?simply-hentai\.com" r"(?!/(?:album|gifs?|images?|series)(?:/|$))" r"((?:/(?!(?:page|all-pages)(?:/|\.|$))[^/?#]+)+)") - test = ( - (("https://original-work.simply-hentai.com" - "/amazon-no-hiyaku-amazon-elixir"), { - "url": "21613585ae5ec2f69ea579e9713f536fceab5bd5", - "keyword": "9e87a0973553b2922ddee37958b8f5d87910af72", - }), - ("https://www.simply-hentai.com/notfound", { - "exception": exception.GalleryDLException, - }), - # custom subdomain - ("https://pokemon.simply-hentai.com/mao-friends-9bc39"), - # www subdomain, two path segments - ("https://www.simply-hentai.com/vocaloid/black-magnet"), - ) + example = "https://www.simply-hentai.com/TITLE" def __init__(self, match): subdomain, path = match.groups() @@ -89,17 +76,7 @@ class SimplyhentaiImageExtractor(Extractor): archive_fmt = "{token}" pattern = (r"(?:https?://)?(?:www\.)?(simply-hentai\.com" r"/(image|gif)/[^/?#]+)") - test = ( - (("https://www.simply-hentai.com/image" - "/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6"), { - "url": "3d8eb55240a960134891bd77fe1df7988fcdc455", - "keyword": "e10e5588481cab68329ef6ec1e5325206b2079a2", - }), - ("https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", { - "url": "f73916527211b4a40f26568ee26cd8999f5f4f30", - "keyword": "f94d775177fed918759c8a78a50976f867425b48", - }), - ) + example = "https://www.simply-hentai.com/image/NAME" def __init__(self, match): Extractor.__init__(self, match) @@ -142,20 +119,7 @@ class SimplyhentaiVideoExtractor(Extractor): filename_fmt = "{title}{episode:?_//>02}.{extension}" archive_fmt = "{title}_{episode}" pattern = r"(?:https?://)?(videos\.simply-hentai\.com/[^/?#]+)" - test = ( - ("https://videos.simply-hentai.com/creamy-pie-episode-02", { - "pattern": r"https://www\.googleapis\.com/drive/v3/files" - r"/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", - "keyword": "706790708b14773efc1e075ddd3b738a375348a5", - "options": (("verify", False),), - "count": 1, - }), - (("https://videos.simply-hentai.com" - "/1715-tifa-in-hentai-gang-bang-3d-movie"), { - "url": "ad9a36ae06c601b6490e3c401834b4949d947eb0", - "keyword": "f9dad94fbde9c95859e631ff4f07297a9567b874", - }), - ) + example = "https://videos.simply-hentai.com/TITLE" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index b643c6f2..55a0db00 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -45,7 +45,7 @@ class SkebExtractor(Extractor): """Return additional metadata""" def _pagination(self, url, params): - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} params["offset"] = 0 while True: @@ -69,7 +69,7 @@ class SkebExtractor(Extractor): def _get_post_data(self, user_name, post_num): url = "{}/api/users/{}/works/{}".format( self.root, user_name, post_num) - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} resp = self.request(url, headers=headers).json() creator = resp["creator"] post = { @@ -145,53 +145,7 @@ class SkebPostExtractor(SkebExtractor): """Extractor for a single skeb post""" subcategory = "post" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/works/(\d+)" - test = ("https://skeb.jp/@kanade_cocotte/works/38", { - "count": 2, - "keyword": { - "anonymous": False, - "body": "re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", - "client": { - "avatar_url": r"re:https://pbs.twimg.com/profile_images" - r"/\d+/\w+\.jpg", - "header_url": r"re:https://pbs.twimg.com/profile_banners" - r"/1375007870291300358/\d+/1500x500", - "id": 1196514, - "name": str, - "screen_name": "minato_ragi", - }, - "content_category": "preview", - "creator": { - "avatar_url": "https://pbs.twimg.com/profile_images" - "/1225470417063645184/P8_SiB0V.jpg", - "header_url": "https://pbs.twimg.com/profile_banners" - "/71243217/1647958329/1500x500", - "id": 159273, - "name": "イチノセ奏", - "screen_name": "kanade_cocotte", - }, - "file_id": int, - "file_url": str, - "genre": "art", - "nsfw": False, - "original": { - "byte_size": int, - "duration": None, - "extension": "re:psd|png", - "frame_rate": None, - "height": 3727, - "is_movie": False, - "width": 2810, - }, - "post_num": "38", - "post_url": "https://skeb.jp/@kanade_cocotte/works/38", - "source_body": None, - "source_thanks": None, - "tags": list, - "thanks": None, - "translated_body": False, - "translated_thanks": None, - } - }) + example = "https://skeb.jp/@USER/works/123" def __init__(self, match): SkebExtractor.__init__(self, match) @@ -205,13 +159,7 @@ class SkebUserExtractor(SkebExtractor): """Extractor for all posts from a skeb user""" subcategory = "user" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/?$" - test = ("https://skeb.jp/@kanade_cocotte", { - "pattern": r"https://skeb\.imgix\.net/uploads/origins/[\w-]+" - r"\?bg=%23fff&auto=format&txtfont=bold&txtshad=70" - r"&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150" - r"&txt=SAMPLE&fm=webp&w=800&s=\w+", - "range": "1-5", - }) + example = "https://skeb.jp/@USER" def posts(self): url = "{}/api/users/{}/works".format(self.root, self.user_name) @@ -230,10 +178,7 @@ class SkebSearchExtractor(SkebExtractor): """Extractor for skeb search results""" subcategory = "search" pattern = r"(?:https?://)?skeb\.jp/search\?q=([^&#]+)" - test = ("https://skeb.jp/search?q=bunny%20tree&t=works", { - "count": ">= 18", - "keyword": {"search_tags": "bunny tree"}, - }) + example = "https://skeb.jp/search?q=QUERY" def metadata(self): return {"search_tags": text.unquote(self.user_name)} @@ -245,7 +190,6 @@ class SkebSearchExtractor(SkebExtractor): } headers = { "Origin": self.root, - "Referer": self.root + "/", "x-algolia-api-key": "9a4ce7d609e71bf29e977925e4c6740c", "x-algolia-application-id": "HB1JT3KRE9", } @@ -286,7 +230,7 @@ class SkebFollowingExtractor(SkebExtractor): """Extractor for all creators followed by a skeb user""" subcategory = "following" pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/following_creators" - test = ("https://skeb.jp/@user/following_creators",) + example = "https://skeb.jp/@USER/following_creators" def items(self): for user in self.users(): @@ -298,7 +242,7 @@ class SkebFollowingExtractor(SkebExtractor): url = "{}/api/users/{}/following_creators".format( self.root, self.user_name) params = {"sort": "date", "offset": 0, "limit": 90} - headers = {"Referer": self.root, "Authorization": "Bearer null"} + headers = {"Authorization": "Bearer null"} while True: data = self.request(url, params=params, headers=headers).json() diff --git a/gallery_dl/extractor/slickpic.py b/gallery_dl/extractor/slickpic.py index 3727c0b0..b2961e17 100644 --- a/gallery_dl/extractor/slickpic.py +++ b/gallery_dl/extractor/slickpic.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -12,7 +12,6 @@ from .common import Extractor, Message from .. import text import time - BASE_PATTERN = r"(?:https?://)?([\w-]+)\.slickpic\.com" @@ -34,22 +33,7 @@ class SlickpicAlbumExtractor(SlickpicExtractor): filename_fmt = "{num:>03}_{id}{title:?_//}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/albums/([^/?#]+)" - test = ( - ("https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", { - "pattern": r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,," - r"/20160807/\w+/p/o/JSBFSS-\d+\.jpg", - "keyword": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", - "count": 102, - }), - ("https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", { - "range": "34", - "content": ("276eb2c902187bb177ae8013e310e1d6641fba9a", - "52b5a310587de1048030ab13a912f6a3a9cc7dab", - "cec6630e659dc72db1ee1a9a6f3b525189261988", - "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", - "22e83645fc242bc3584eca7ec982c8a53a4d8a44"), - }), - ) + example = "https://USER.slickpic.com/albums/TITLE/" def __init__(self, match): SlickpicExtractor.__init__(self, match) @@ -127,13 +111,7 @@ class SlickpicAlbumExtractor(SlickpicExtractor): class SlickpicUserExtractor(SlickpicExtractor): subcategory = "user" pattern = BASE_PATTERN + r"(?:/gallery)?/?(?:$|[?#])" - test = ( - ("https://mattcrandall.slickpic.com/gallery/", { - "count": ">= 358", - "pattern": SlickpicAlbumExtractor.pattern, - }), - ("https://mattcrandall.slickpic.com/"), - ) + example = "https://USER.slickpic.com/" def items(self): page = self.request(self.root + "/gallery?viewer").text diff --git a/gallery_dl/extractor/slideshare.py b/gallery_dl/extractor/slideshare.py index 3521298f..b56ed27b 100644 --- a/gallery_dl/extractor/slideshare.py +++ b/gallery_dl/extractor/slideshare.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2023 Mike Fährmann, Leonardo Taccari +# Copyright 2016-2017 Leonardo Taccari +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,45 +22,7 @@ class SlidesharePresentationExtractor(GalleryExtractor): archive_fmt = "{presentation}_{num}" pattern = (r"(?:https?://)?(?:www\.)?slideshare\.net" r"/(?:mobile/)?([^/?#]+)/([^/?#]+)") - test = ( - (("https://www.slideshare.net" - "/Slideshare/get-started-with-slide-share"), { - "pattern": r"https://image\.slidesharecdn\.com/getstartedwithslide" - r"share-150520173821-lva1-app6892/95/get-started-with-s" - r"lide-share-\d+-1024\.jpg\?cb=\d+", - "count": 19, - "content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", - "keyword": { - "description": "Get Started with SlideShare - " - "A Beginngers Guide for Creators", - "likes": int, - "presentation": "get-started-with-slide-share", - "date": "dt:2015-05-20 17:38:21", - "title": "Getting Started With SlideShare", - "user": "Slideshare", - "views": int, - }, - }), - # long title and description - (("https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren" - "-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem"), { - "url": "d8952260f8bec337dd809a958ec8091350393f6b", - "keyword": { - "title": "Warum Sie nicht Ihren Mitarbeitenden ändern " - "sollten, sondern Ihr Managementsystem", - "description": "Mitarbeitende verhalten sich mehrheitlich so, " - "wie das System es ihnen vorgibt. Welche Voraus" - "setzungen es braucht, damit Ihre Mitarbeitende" - "n ihr ganzes Herzblut einsetzen, bespricht Fre" - "di Schmidli in diesem Referat.", - }, - }), - # mobile URL - (("https://www.slideshare.net" - "/mobile/uqudent/introduction-to-fixed-prosthodontics"), { - "url": "72c431cb1eccbb6794f608ecbbc01d52e8768159", - }), - ) + example = "https://www.slideshare.net/USER/PRESENTATION" def __init__(self, match): self.user, self.presentation = match.groups() diff --git a/gallery_dl/extractor/smugmug.py b/gallery_dl/extractor/smugmug.py index b9edd4ab..3639c0b5 100644 --- a/gallery_dl/extractor/smugmug.py +++ b/gallery_dl/extractor/smugmug.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2022 Mike Fährmann +# Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -68,19 +68,7 @@ class SmugmugAlbumExtractor(SmugmugExtractor): directory_fmt = ("{category}", "{User[NickName]}", "{Album[Name]}") archive_fmt = "a_{Album[AlbumKey]}_{Image[ImageKey]}" pattern = r"smugmug:album:([^:]+)$" - test = ( - ("smugmug:album:cr4C7f", { - "url": "2c2e576e47d4e9ce60b44871f08a8c66b5ebaceb", - }), - # empty - ("smugmug:album:Fb7hMs", { - "count": 0, - }), - # no "User" - ("smugmug:album:6VRT8G", { - "url": "c4a0f4c4bfd514b93cbdeb02b3345bf7ef6604df", - }), - ) + example = "smugmug:album:ID" def __init__(self, match): SmugmugExtractor.__init__(self, match) @@ -107,18 +95,7 @@ class SmugmugImageExtractor(SmugmugExtractor): subcategory = "image" archive_fmt = "{Image[ImageKey]}" pattern = BASE_PATTERN + r"(?:/[^/?#]+)+/i-([^/?#-]+)" - test = ( - ("https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", { - "url": "e6408fd2c64e721fd146130dceb56a971ceb4259", - "keyword": "b31a63d07c9c26eb0f79f52d60d171a98938f99b", - "content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", - }), - # video - ("https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", { - "url": "04d0ab1ff829ca7d78f5acb5548953df08e9a5ee", - "keyword": "2b545184592c282b365fcbb7df6ca7952b8a3173", - }), - ) + example = "https://USER.smugmug.com/PATH/i-ID" def __init__(self, match): SmugmugExtractor.__init__(self, match) @@ -139,35 +116,13 @@ class SmugmugPathExtractor(SmugmugExtractor): """Extractor for smugmug albums from URL paths and users""" subcategory = "path" pattern = BASE_PATTERN + r"((?:/[^/?#a-fh-mo-z][^/?#]*)*)/?$" - test = ( - ("https://tdm.smugmug.com/Nature/Dove", { - "pattern": "smugmug:album:cr4C7f$", - }), - ("https://tdm.smugmug.com/", { - "pattern": SmugmugAlbumExtractor.pattern, - "url": "1640028712875b90974e5aecd91b60e6de6138c7", - }), - # gallery node without owner - ("https://www.smugmug.com/gallery/n-GLCjnD/", { - "pattern": "smugmug:album:6VRT8G$", - }), - # custom domain - ("smugmug:www.sitkapics.com/TREES-and-TRAILS/", { - "pattern": "smugmug:album:ct8Nds$", - }), - ("smugmug:www.sitkapics.com/", { - "pattern": r"smugmug:album:\w{6}$", - "count": ">= 14", - }), - ("smugmug:https://www.sitkapics.com/"), - ) + example = "https://USER.smugmug.com/PATH" def __init__(self, match): SmugmugExtractor.__init__(self, match) self.domain, self.user, self.path = match.groups() def items(self): - if not self.user: self.user = self.api.site_user(self.domain)["NickName"] diff --git a/gallery_dl/extractor/soundgasm.py b/gallery_dl/extractor/soundgasm.py index 236f94f6..7c75aaae 100644 --- a/gallery_dl/extractor/soundgasm.py +++ b/gallery_dl/extractor/soundgasm.py @@ -51,29 +51,7 @@ class SoundgasmAudioExtractor(SoundgasmExtractor): """Extractor for audio clips from soundgasm.net""" subcategory = "audio" pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)" - test = ( - (("https://soundgasm.net/u/ClassWarAndPuppies2" - "/687-Otto-von-Toontown-12822"), { - "pattern": r"https://media\.soundgasm\.net/sounds" - r"/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", - "keyword": { - "description": "We celebrate today’s important prisoner swap, " - "and finally bring the 2022 mid-terms to a clos" - "e with Raphael Warnock’s defeat of Herschel Wa" - "lker in Georgia. Then, we take a look at the Q" - "anon-addled attempt to overthrow the German go" - "vernment and install Heinrich XIII Prince of R" - "euss as kaiser.", - "extension": "m4a", - "filename": "26cb2b23b2f2c6094b40ee3a9167271e274b570a", - "slug": "687-Otto-von-Toontown-12822", - "title": "687 - Otto von Toontown (12/8/22)", - "user": "ClassWarAndPuppies2", - }, - }), - ("https://www.soundgasm.net/user/ClassWarAndPuppies2" - "/687-Otto-von-Toontown-12822"), - ) + example = "https://soundgasm.net/u/USER/TITLE" def __init__(self, match): SoundgasmExtractor.__init__(self, match) @@ -87,19 +65,7 @@ class SoundgasmUserExtractor(SoundgasmExtractor): """Extractor for all sounds from a soundgasm user""" subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)/?$" - test = ("https://soundgasm.net/u/fierce-aphrodite", { - "pattern": r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", - "count" : ">= 15", - "keyword": { - "description": str, - "extension": "m4a", - "filename": "re:^[0-9a-f]{40}$", - "slug": str, - "title": str, - "url": str, - "user": "fierce-aphrodite" - }, - }) + example = "https://soundgasm.net/u/USER" def __init__(self, match): SoundgasmExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/speakerdeck.py b/gallery_dl/extractor/speakerdeck.py index 9eb87617..e44fdae7 100644 --- a/gallery_dl/extractor/speakerdeck.py +++ b/gallery_dl/extractor/speakerdeck.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://speakerdeck.com/""" +"""Extractors for https://speakerdeck.com/""" from .common import Extractor, Message from .. import text @@ -19,16 +19,8 @@ class SpeakerdeckPresentationExtractor(Extractor): directory_fmt = ("{category}", "{user}") filename_fmt = "{presentation}-{num:>02}.{extension}" archive_fmt = "{presentation}_{num}" - pattern = (r"(?:https?://)?(?:www\.)?speakerdeck\.com" - r"/([^/?#]+)/([^/?#]+)") - test = ( - (("https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck"), { - "pattern": r"https://files.speakerdeck.com/presentations/" - r"50021f75cf1db900020005e7/slide_\d+.jpg", - "content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", - "count": 6, - }), - ) + pattern = r"(?:https?://)?(?:www\.)?speakerdeck\.com/([^/?#]+)/([^/?#]+)" + example = "https://speakerdeck.com/USER/PRESENTATION" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py index a2e1388c..6b4cba21 100644 --- a/gallery_dl/extractor/subscribestar.py +++ b/gallery_dl/extractor/subscribestar.py @@ -84,16 +84,16 @@ class SubscribestarExtractor(Extractor): if cookie.name.startswith("auth") } - @staticmethod - def _media_from_post(html): + def _media_from_post(self, html): media = [] gallery = text.extr(html, 'data-gallery="', '"') if gallery: - media.extend( - item for item in util.json_loads(text.unescape(gallery)) - if "/previews/" not in item["url"] - ) + for item in util.json_loads(text.unescape(gallery)): + if "/previews" in item["url"]: + self._warn_preview() + else: + media.append(item) attachments = text.extr( html, 'class="uploads-docs"', 'data-role="post-edit_form"') @@ -130,39 +130,16 @@ class SubscribestarExtractor(Extractor): date = text.parse_datetime(dt, "%B %d, %Y %I:%M %p") return date + def _warn_preview(self): + self.log.warning("Preview image detected") + self._warn_preview = util.noop + class SubscribestarUserExtractor(SubscribestarExtractor): """Extractor for media from a subscribestar user""" subcategory = "user" pattern = BASE_PATTERN + r"/(?!posts/)([^/?#]+)" - test = ( - ("https://www.subscribestar.com/subscribestar", { - "count": ">= 20", - "pattern": r"https://\w+\.cloudfront\.net/uploads(_v2)?/users/11/", - "keyword": { - "author_id": 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content": str, - "date" : "type:datetime", - "id" : int, - "num" : int, - "post_id": int, - "type" : "re:image|video|attachment", - "url" : str, - "?pinned": bool, - }, - }), - ("https://www.subscribestar.com/subscribestar", { - "options": (("metadata", True),), - "keyword": {"date": "type:datetime"}, - "range": "1", - }), - ("https://subscribestar.adult/kanashiipanda", { - "range": "1-10", - "count": 10, - }), - ) + example = "https://www.subscribestar.com/USER" def posts(self): needle_next_page = 'data-role="infinite_scroll-next_page" href="' @@ -184,32 +161,7 @@ class SubscribestarPostExtractor(SubscribestarExtractor): """Extractor for media from a single subscribestar post""" subcategory = "post" pattern = BASE_PATTERN + r"/posts/(\d+)" - test = ( - ("https://www.subscribestar.com/posts/102468", { - "count": 1, - "keyword": { - "author_id": 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content": "re:

Brand Guidelines and Assets

", - "date": "dt:2020-05-07 12:33:00", - "extension": "jpg", - "filename": "8ff61299-b249-47dc-880a-cdacc9081c62", - "group": "imgs_and_videos", - "height": 291, - "id": 203885, - "num": 1, - "pinned": False, - "post_id": 102468, - "type": "image", - "width": 700, - }, - }), - ("https://subscribestar.adult/posts/22950", { - "count": 1, - "keyword": {"date": "dt:2019-04-28 07:32:00"}, - }), - ) + example = "https://www.subscribestar.com/posts/12345" def posts(self): url = "{}/posts/{}".format(self.root, self.item) diff --git a/gallery_dl/extractor/szurubooru.py b/gallery_dl/extractor/szurubooru.py index 8c816ad1..5415bf30 100644 --- a/gallery_dl/extractor/szurubooru.py +++ b/gallery_dl/extractor/szurubooru.py @@ -95,15 +95,7 @@ class SzurubooruTagExtractor(SzurubooruExtractor): directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}_{version}" pattern = BASE_PATTERN + r"/posts/query=([^/?#]+)" - test = ( - ("https://booru.foalcon.com/posts/query=simple_background", { - "pattern": r"https://booru\.foalcon\.com/data/posts" - r"/\d+_[0-9a-f]{16}\.\w+", - "range": "1-150", - "count": 150, - }), - ("https://booru.bcbnsfw.space/posts/query=simple_background"), - ) + example = "https://booru.foalcon.com/posts/query=TAG" def __init__(self, match): SzurubooruExtractor.__init__(self, match) @@ -121,19 +113,7 @@ class SzurubooruPostExtractor(SzurubooruExtractor): subcategory = "post" archive_fmt = "{id}_{version}" pattern = BASE_PATTERN + r"/post/(\d+)" - test = ( - ("https://booru.foalcon.com/post/30092", { - "pattern": r"https://booru\.foalcon\.com/data/posts" - r"/30092_b7d56e941888b624\.png", - "url": "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", - "content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", - }), - ("https://booru.bcbnsfw.space/post/1599", { - "pattern": r"https://booru\.bcbnsfw\.space/data/posts" - r"/1599_53784518e92086bd\.png", - "content": "0c38fc612ba1f03950fad31c4f80a1fccdab1096", - }), - ) + example = "https://booru.foalcon.com/post/12345" def __init__(self, match): SzurubooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/tapas.py b/gallery_dl/extractor/tapas.py index 0e09e22a..bfca7a62 100644 --- a/gallery_dl/extractor/tapas.py +++ b/gallery_dl/extractor/tapas.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -107,15 +107,7 @@ class TapasExtractor(Extractor): class TapasSeriesExtractor(TapasExtractor): subcategory = "series" pattern = BASE_PATTERN + r"/series/([^/?#]+)" - test = ( - ("https://tapas.io/series/just-leave-me-be", { - "pattern": r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", - "count": 132, - }), - ("https://tapas.io/series/yona", { # mature - "count": 26, - }), - ) + example = "https://tapas.io/series/TITLE" def __init__(self, match): TapasExtractor.__init__(self, match) @@ -151,53 +143,7 @@ class TapasSeriesExtractor(TapasExtractor): class TapasEpisodeExtractor(TapasExtractor): subcategory = "episode" pattern = BASE_PATTERN + r"/episode/(\d+)" - test = ("https://tapas.io/episode/2068651", { - "url": "0b53644c864a0a097f65accea6bb620be9671078", - "pattern": "^text:", - "keyword": { - "book": True, - "comment_cnt": int, - "date": "dt:2021-02-23 16:02:07", - "early_access": False, - "escape_title": "You are a Tomb Raider (2)", - "free": True, - "id": 2068651, - "like_cnt": int, - "liked": bool, - "mature": False, - "next_ep_id": 2068652, - "nsfw": False, - "nu": False, - "num": 1, - "open_comments": True, - "pending_scene": 2, - "prev_ep_id": 2068650, - "publish_date": "2021-02-23T16:02:07Z", - "read": bool, - "related_ep_id": None, - "relative_publish_date": "Feb 23, 2021", - "scene": 2, - "scheduled": False, - "title": "You are a Tomb Raider (2)", - "unlock_cnt": 0, - "unlocked": False, - "view_cnt": int, - - "series": { - "genre": dict, - "has_book_cover": True, - "has_top_banner": True, - "id": 199931, - "premium": True, - "sale_type": "PAID", - "subscribed": bool, - "thumbsup_cnt": int, - "title": "Tomb Raider King", - "type": "BOOKS", - "url": "tomb-raider-king-novel", - }, - }, - }) + example = "https://tapas.io/episode/12345" def __init__(self, match): TapasExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/tcbscans.py b/gallery_dl/extractor/tcbscans.py index b5a730a4..a3ef26cb 100644 --- a/gallery_dl/extractor/tcbscans.py +++ b/gallery_dl/extractor/tcbscans.py @@ -16,35 +16,7 @@ class TcbscansChapterExtractor(ChapterExtractor): category = "tcbscans" root = "https://tcbscans.com" pattern = BASE_PATTERN + r"(/chapters/\d+/[^/?#]+)" - test = ( - (("https://tcbscans.com" - "/chapters/4708/chainsaw-man-chapter-108"), { - "pattern": (r"https://cdn\.[^/]+" - r"/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+"), - "count" : 17, - "keyword": { - "manga": "Chainsaw Man", - "chapter": 108, - "chapter_minor": "", - "lang": "en", - "language": "English", - }, - }), - ("https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", { - "pattern": (r"https://cdn\.[^/]+" - r"/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+"), - "count" : 18, - "keyword": { - "manga": "One Piece", - "chapter": 1065, - "chapter_minor": "", - "lang": "en", - "language": "English", - }, - }), - (("https://onepiecechapters.com/" - "chapters/44/ace-novel-manga-adaptation-chapter-1")), - ) + example = "https://tcbscans.com/chapters/12345/MANGA-chapter-123" def images(self, page): return [ @@ -70,19 +42,7 @@ class TcbscansMangaExtractor(MangaExtractor): root = "https://tcbscans.com" chapterclass = TcbscansChapterExtractor pattern = BASE_PATTERN + r"(/mangas/\d+/[^/?#]+)" - test = ( - ("https://tcbscans.com/mangas/13/chainsaw-man", { - "pattern": TcbscansChapterExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://onepiecechapters.com/mangas/4/jujutsu-kaisen", { - "pattern": TcbscansChapterExtractor.pattern, - "range" : "1-50", - "count" : 50, - }), - ("https://onepiecechapters.com/mangas/15/hunter-x-hunter"), - ) + example = "https://tcbscans.com/mangas/123/MANGA" def chapters(self, page): data = { diff --git a/gallery_dl/extractor/telegraph.py b/gallery_dl/extractor/telegraph.py index 116f3afa..dd5988f8 100644 --- a/gallery_dl/extractor/telegraph.py +++ b/gallery_dl/extractor/telegraph.py @@ -18,72 +18,7 @@ class TelegraphGalleryExtractor(GalleryExtractor): filename_fmt = "{num_formatted}_{filename}.{extension}" archive_fmt = "{slug}_{num}" pattern = r"(?:https?://)(?:www\.)??telegra\.ph(/[^/?#]+)" - test = ( - ("https://telegra.ph/Telegraph-Test-03-28", { - "pattern": r"https://telegra\.ph/file/[0-9a-f]+\.png", - "keyword": { - "author": "mikf", - "caption": r"re:test|", - "count": 2, - "date": "dt:2022-03-28 16:01:36", - "description": "Just a test", - "post_url": "https://telegra.ph/Telegraph-Test-03-28", - "slug": "Telegraph-Test-03-28", - "title": "Telegra.ph Test", - }, - }), - ("https://telegra.ph/森-03-28", { - "pattern": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - "count": 1, - "keyword": { - "author": "&", - "caption": "kokiri", - "count": 1, - "date": "dt:2022-03-28 16:31:26", - "description": "コキリの森", - "extension": "jpg", - "filename": "3ea79d23b0dd0889f215a", - "num": 1, - "num_formatted": "1", - "post_url": "https://telegra.ph/森-03-28", - "slug": "森-03-28", - "title": '"森"', - "url": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - }, - }), - ("https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", { - "url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", - "pattern": r"^https://pith1\.ru/uploads" - r"/posts/2019-12/\d+_\d+\.jpg$", - "keyword": { - "author": "Shotacon - заходи сюда", - "caption": "", - "count": 19, - "date": "dt:2022-05-27 16:17:27", - "description": "", - "num_formatted": r"re:^\d{2}$", - "post_url": "https://telegra.ph" - "/Vsyo-o-druzyah-moej-sestricy-05-27", - "slug": "Vsyo-o-druzyah-moej-sestricy-05-27", - "title": "Всё о друзьях моей сестрицы", - }, - }), - ("https://telegra.ph/Disharmonica---Saber-Nero-02-21", { - "pattern": r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", - "keyword": { - "author": "cosmos", - "caption": "", - "count": 89, - "date": "dt:2022-02-21 05:57:39", - "description": "", - "num_formatted": r"re:^\d{2}$", - "post_url": "https://telegra.ph" - "/Disharmonica---Saber-Nero-02-21", - "slug": "Disharmonica---Saber-Nero-02-21", - "title": "Disharmonica - Saber Nero", - }, - }), - ) + example = "https://telegra.ph/TITLE" def metadata(self, page): extr = text.extract_from(page) diff --git a/gallery_dl/extractor/test.py b/gallery_dl/extractor/test.py index 5d81ed51..e3f9f744 100644 --- a/gallery_dl/extractor/test.py +++ b/gallery_dl/extractor/test.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2017 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -33,11 +33,7 @@ class TestExtractor(Extractor): """ category = "test" pattern = r"t(?:est)?:([^:]*)(?::([^:]*)(?::(\*|[\d,]*))?)?$" - test = ( - ("test:pixiv"), - ("test:pixiv:user,favorite:0"), - ("test:"), - ) + example = "test:CATEGORY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index c6be38dd..78ff265d 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -99,28 +99,7 @@ class ToyhouseArtExtractor(ToyhouseExtractor): """Extractor for artworks of a toyhouse user""" subcategory = "art" pattern = BASE_PATTERN + r"/([^/?#]+)/art" - - test = ( - ("https://www.toyhou.se/d-floe/art", { - "range": "1-30", - "count": 30, - "pattern": r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se" - r"/images/\d+_\w+\.\w+$", - "keyword": { - "artists": list, - "characters": list, - "date": "type:datetime", - "hash": r"re:\w+", - "id": r"re:\d+", - "url": str, - "user": "d-floe", - }, - }), - # protected by Content Warning - ("https://www.toyhou.se/kroksoc/art", { - "count": ">= 19", - }), - ) + example = "https://www.toyhou.se/USER/art" def posts(self): return self._pagination("/{}/art".format(self.user)) @@ -136,37 +115,7 @@ class ToyhouseImageExtractor(ToyhouseExtractor): r"(?:www\.)?toyhou\.se/~images|" r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" r")/(\d+)") - test = ( - ("https://toyhou.se/~images/40587320", { - "content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", - "keyword": { - "artists": ["d-floe"], - "characters": ["Sumi"], - "date": "dt:2021-10-08 01:32:47", - "extension": "png", - "filename": "40587320_TT1NaBUr3FLkS1p", - "hash": "TT1NaBUr3FLkS1p", - "id": "40587320", - "url": "https://f2.toyhou.se/file/f2-toyhou-se/images" - "/40587320_TT1NaBUr3FLkS1p.png", - }, - }), - # direct link, multiple artists - (("https://f2.toyhou.se/file/f2-toyhou-se" - "/watermarks/36817425_bqhGcwcnU.png?1625561467"), { - "keyword": { - "artists": [ - "http://aminoapps.com/p/92sf3z", - "kroksoc (Color)"], - "characters": ["❀Reiichi❀"], - "date": "dt:2021-07-03 20:02:02", - "hash": "bqhGcwcnU", - "id": "36817425", - }, - }), - ("https://f2.toyhou.se/file/f2-toyhou-se" - "/images/40587320_TT1NaBUr3FLkS1p.png"), - ) + example = "https://toyhou.se/~images/12345" def posts(self): url = "{}/~images/{}".format(self.root, self.user) diff --git a/gallery_dl/extractor/tsumino.py b/gallery_dl/extractor/tsumino.py index e7d5226a..de7cdfc0 100644 --- a/gallery_dl/extractor/tsumino.py +++ b/gallery_dl/extractor/tsumino.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -44,34 +44,7 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): """Extractor for image galleries on tsumino.com""" pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com" r"/(?:entry|Book/Info|Read/(?:Index|View))/(\d+)") - test = ( - ("https://www.tsumino.com/entry/40996", { - "pattern": r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", - "keyword": { - "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", - "gallery_id": 40996, - "date" : "dt:2018-06-29 00:00:00", - "count" : 42, - "collection": "", - "artist" : ["Itou Life"], - "group" : ["Itou Life"], - "parody" : list, - "characters": list, - "tags" : list, - "type" : "Doujinshi", - "rating" : float, - "uploader" : "sehki", - "lang" : "en", - "language" : "English", - "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", - }, - }), - ("https://www.tsumino.com/Book/Info/40996"), - ("https://www.tsumino.com/Read/View/45834"), - ("https://www.tsumino.com/Read/Index/45834"), - ) + example = "https://www.tsumino.com/entry/12345" def __init__(self, match): self.gallery_id = match.group(1) @@ -131,21 +104,8 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor): class TsuminoSearchExtractor(TsuminoBase, Extractor): """Extractor for search results on tsumino.com""" subcategory = "search" - pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com" - r"/(?:Books/?)?#(.+)") - test = ( - ("https://www.tsumino.com/Books#?Character=Reimu+Hakurei", { - "pattern": TsuminoGalleryExtractor.pattern, - "range": "1-40", - "count": 40, - }), - (("http://www.tsumino.com/Books#~(Tags~(~" - "(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~" - "(Type~'1~Text~'Pantyhose~Exclude~false)))#"), { - "pattern": TsuminoGalleryExtractor.pattern, - "count": ">= 3", - }), - ) + pattern = r"(?i)(?:https?://)?(?:www\.)?tsumino\.com/(?:Books/?)?#(.+)" + example = "https://www.tsumino.com/Books#QUERY" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 9adc3ab1..3dab16e8 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -273,59 +273,7 @@ class TumblrUserExtractor(TumblrExtractor): """Extractor for a Tumblr user's posts""" subcategory = "user" pattern = BASE_PATTERN + r"(?:/page/\d+|/archive)?/?$" - test = ( - ("http://demo.tumblr.com/", { - "pattern": r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_\d+\.jpg", - "count": 1, - "options": (("posts", "photo"),), - }), - ("http://demo.tumblr.com/", { - "pattern": (r"https?://(?:$|" - r"\d+\.media\.tumblr\.com/.+_1280\.jpg|" - r"a\.tumblr\.com/tumblr_\w+)"), - "count": 3, - "options": (("posts", "all"), ("external", True)) - }), - ("https://mikf123-hidden.tumblr.com/", { # dashboard-only - "options": (("access-token", None),), - "exception": exception.AuthorizationError, - }), - ("https://mikf123-hidden.tumblr.com/", { # dashboard-only - "count": 2, - "keyword": {"tags": ["test", "hidden"]}, - }), - ("https://mikf123-private.tumblr.com/", { # password protected - "count": 2, - "keyword": {"tags": ["test", "private"]}, - }), - ("https://mikf123-private-hidden.tumblr.com/", { # both - "count": 2, - "keyword": {"tags": ["test", "private", "hidden"]}, - }), - ("https://mikf123.tumblr.com/", { # date-min/-max/-format (#337) - "count": 4, - "options": (("date-min", "201804"), ("date-max", "201805"), - ("date-format", "%Y%m")) - }), - # pagination with 'date-max' (#2191) and 'api-key' - ("https://donttrustthetits.tumblr.com/", { - "options": ( - ("access-token", None), - ("original", False), - ("date-max", "2015-04-25T00:00:00"), - ("date-min", "2015-04-01T00:00:00"), - ), - "count": 316, - }), - ("https://demo.tumblr.com/page/2"), - ("https://demo.tumblr.com/archive"), - ("tumblr:http://www.b-authentique.com/"), - ("tumblr:www.b-authentique.com"), - ("https://www.tumblr.com/blog/view/smarties-art"), - ("https://www.tumblr.com/blog/smarties-art"), - ("https://www.tumblr.com/smarties-art"), - ) + example = "https://www.tumblr.com/BLOG" def posts(self): return self.api.posts(self.blog, {}) @@ -335,55 +283,7 @@ class TumblrPostExtractor(TumblrExtractor): """Extractor for a single Tumblr post""" subcategory = "post" pattern = BASE_PATTERN + r"/(?:post/|image/)?(\d+)" - test = ( - ("http://demo.tumblr.com/post/459265350", { - "pattern": (r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_1280.jpg"), - "count": 1, - }), - ("https://mikf123.tumblr.com/post/167770226574/text-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/181022561719/quote-post", { - "count": 1, - }), - ("https://mikf123.tumblr.com/post/167623351559/link-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/167633596145/video-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/167770026604/audio-post", { - "count": 2, - }), - ("https://mikf123.tumblr.com/post/172687798174/photo-post", { - "count": 4, - }), - ("https://mikf123.tumblr.com/post/181022380064/chat-post", { - "count": 0, - }), - ("https://kichatundk.tumblr.com/post/654953419288821760", { - "count": 2, # high-quality images (#1846) - "content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", - }), - ("https://hameru-is-cool.tumblr.com/post/639261855227002880", { - "count": 2, # high-quality images (#1344) - "content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", - }), - ("https://mikf123.tumblr.com/image/689860196535762944", { - "pattern": r"^https://\d+\.media\.tumblr\.com" - r"/134791621559a79793563b636b5fe2c6" - r"/8f1131551cef6e74-bc/s99999x99999" - r"/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", - }), - ("http://ziemniax.tumblr.com/post/109697912859/", { - "exception": exception.NotFoundError, # HTML response (#297) - }), - ("http://demo.tumblr.com/image/459265350"), - ("https://www.tumblr.com/blog/view/smarties-art/686047436641353728"), - ("https://www.tumblr.com/blog/smarties-art/686047436641353728"), - ("https://www.tumblr.com/smarties-art/686047436641353728"), - ) + example = "https://www.tumblr.com/BLOG/12345" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -403,16 +303,7 @@ class TumblrTagExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by tag""" subcategory = "tag" pattern = BASE_PATTERN + r"/tagged/([^/?#]+)" - test = ( - ("http://demo.tumblr.com/tagged/Times%20Square", { - "pattern": r"https://\d+\.media\.tumblr\.com" - r"/tumblr_[^/_]+_1280.jpg", - "count": 1, - }), - ("https://www.tumblr.com/blog/view/smarties-art/tagged/undertale"), - ("https://www.tumblr.com/blog/smarties-art/tagged/undertale"), - ("https://www.tumblr.com/smarties-art/tagged/undertale"), - ) + example = "https://www.tumblr.com/BLOG/tagged/TAG" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -426,18 +317,7 @@ class TumblrDayExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by day""" subcategory = "day" pattern = BASE_PATTERN + r"/day/(\d\d\d\d/\d\d/\d\d)" - test = ( - ("https://mikf123.tumblr.com/day/2018/01/05", { - "pattern": r"https://64\.media\.tumblr\.com" - r"/1a2be8c63f1df58abd2622861696c72a" - r"/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", - "keyword": {"id": 169341068404}, - "count": 1, - }), - ("https://www.tumblr.com/blog/view/mikf123/day/2018/01/05"), - ("https://www.tumblr.com/blog/mikf123/day/2018/01/05"), - ("https://www.tumblr.com/mikf123/day/2018/01/05"), - ) + example = "https://www.tumblr.com/BLOG/day/1970/01/01" def __init__(self, match): TumblrExtractor.__init__(self, match) @@ -460,14 +340,7 @@ class TumblrLikesExtractor(TumblrExtractor): directory_fmt = ("{category}", "{blog_name}", "likes") archive_fmt = "f_{blog[name]}_{id}_{num}" pattern = BASE_PATTERN + r"/likes" - test = ( - ("http://mikf123.tumblr.com/likes", { - "count": 1, - }), - ("https://www.tumblr.com/blog/view/mikf123/likes"), - ("https://www.tumblr.com/blog/mikf123/likes"), - ("https://www.tumblr.com/mikf123/likes"), - ) + example = "https://www.tumblr.com/BLOG/likes" def posts(self): return self.api.likes(self.blog) diff --git a/gallery_dl/extractor/tumblrgallery.py b/gallery_dl/extractor/tumblrgallery.py index 6940f3e6..27cc9d0c 100644 --- a/gallery_dl/extractor/tumblrgallery.py +++ b/gallery_dl/extractor/tumblrgallery.py @@ -38,7 +38,7 @@ class TumblrgalleryTumblrblogExtractor(TumblrgalleryExtractor): """Extractor for Tumblrblog on tumblrgallery.xyz""" subcategory = "tumblrblog" pattern = BASE_PATTERN + r"(/tumblrblog/gallery/(\d+)\.html)" - test = ("https://tumblrgallery.xyz/tumblrblog/gallery/103975.html",) + example = "https://tumblrgallery.xyz/tumblrblog/gallery/12345.html" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) @@ -69,11 +69,7 @@ class TumblrgalleryPostExtractor(TumblrgalleryExtractor): """Extractor for Posts on tumblrgallery.xyz""" subcategory = "post" pattern = BASE_PATTERN + r"(/post/(\d+)\.html)" - test = ("https://tumblrgallery.xyz/post/405674.html", { - "pattern": r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711d" - r"ec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", - "count": 3, - }) + example = "https://tumblrgallery.xyz/post/12345.html" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) @@ -98,10 +94,7 @@ class TumblrgallerySearchExtractor(TumblrgalleryExtractor): filename_fmt = "{category}_{num:>03}_{gallery_id}_{id}_{title}.{extension}" directory_fmt = ("{category}", "{search_term}") pattern = BASE_PATTERN + r"(/s\.php\?q=([^&#]+))" - test = ("https://tumblrgallery.xyz/s.php?q=everyday-life", { - "pattern": r"https://\d+\.media\.tumblr\.com/.+", - "count": "< 1000", - }) + example = "https://tumblrgallery.xyz/s.php?q=QUERY" def __init__(self, match): TumblrgalleryExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/twibooru.py b/gallery_dl/extractor/twibooru.py index c3e0a262..49c84195 100644 --- a/gallery_dl/extractor/twibooru.py +++ b/gallery_dl/extractor/twibooru.py @@ -19,12 +19,12 @@ class TwibooruExtractor(BooruExtractor): """Base class for twibooru extractors""" category = "twibooru" basecategory = "philomena" + root = "https://twibooru.org" filename_fmt = "{id}_{filename}.{extension}" archive_fmt = "{id}" request_interval = 6.05 page_start = 1 per_page = 50 - root = "https://twibooru.org" def _init(self): self.api = TwibooruAPI(self) @@ -46,50 +46,7 @@ class TwibooruPostExtractor(TwibooruExtractor): subcategory = "post" request_interval = 1.0 pattern = BASE_PATTERN + r"/(\d+)" - test = ("https://twibooru.org/1", { - "pattern": r"https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", - "keyword": { - "animated": False, - "aspect_ratio": 1.0, - "comment_count": int, - "created_at": "2020-07-08T22:26:55.743Z", - "date": "dt:2020-07-08 22:26:55", - "description": "Why have I done this?", - "downvotes": 0, - "duration": 0.0, - "faves": int, - "first_seen_at": "2020-07-08T22:26:55.743Z", - "format": "png", - "height": 576, - "hidden_from_users": False, - "id": 1, - "intensities": dict, - "locations": [], - "media_type": "image", - "mime_type": "image/png", - "name": "1676547__safe_artist-colon-scraggleman_oc_oc-colon-" - "floor+bored_oc+only_bags+under+eyes_bust_earth+pony_" - "female_goggles_helmet_mare_meme_neet_neet+home+g.png", - "orig_sha512_hash": "re:8b4c00d2[0-9a-f]{120}", - "processed": True, - "representations": dict, - "score": int, - "sha512_hash": "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf6" - "34fbb181b32f44070aa588938e26c4eb072b1eb61489aa" - "f3062fb644a76c79f936b97723a2c3e0e5d3", - "size": 70910, - "source_url": "", - "tag_ids": list, - "tags": list, - "thumbnails_generated": True, - "updated_at": "2022-11-27T00:34:50.483Z", - "upvotes": int, - "view_url": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "width": 576, - "wilson_score": float, - }, - }) + example = "https://twibooru.org/12345" def __init__(self, match): TwibooruExtractor.__init__(self, match) @@ -104,16 +61,7 @@ class TwibooruSearchExtractor(TwibooruExtractor): subcategory = "search" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?:search/?\?([^#]+)|tags/([^/?#]+))" - test = ( - ("https://twibooru.org/search?q=cute", { - "range": "40-60", - "count": 21, - }), - ("https://twibooru.org/tags/cute", { - "range": "1-20", - "count": 20, - }), - ) + example = "https://twibooru.org/search?q=TAG" def __init__(self, match): TwibooruExtractor.__init__(self, match) @@ -147,19 +95,7 @@ class TwibooruGalleryExtractor(TwibooruExtractor): directory_fmt = ("{category}", "galleries", "{gallery[id]} {gallery[title]}") pattern = BASE_PATTERN + r"/galleries/(\d+)" - test = ("https://twibooru.org/galleries/1", { - "range": "1-20", - "keyword": { - "gallery": { - "description": "Best nation pone and " - "russian related pics.", - "id": 1, - "spoiler_warning": "Russia", - "thumbnail_id": 694923, - "title": "Marussiaverse", - }, - }, - }) + example = "https://twibooru.org/galleries/12345" def __init__(self, match): TwibooruExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 81b5291b..3895c74c 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -277,6 +277,8 @@ class TwitterExtractor(Extractor): else: note = None + source = tweet["source"] + if "legacy" in tweet: tweet = tweet["legacy"] @@ -303,6 +305,7 @@ class TwitterExtractor(Extractor): "author" : author, "user" : self._user or author, "lang" : tweet["lang"], + "source" : text.extr(source, ">", "<"), "favorite_count": tget("favorite_count"), "quote_count" : tget("quote_count"), "reply_count" : tget("reply_count"), @@ -336,6 +339,9 @@ class TwitterExtractor(Extractor): tdata["reply_to"] = tweet["in_reply_to_screen_name"] if "quoted_by" in tweet: tdata["quote_by"] = tweet["quoted_by"] + if tdata["retweet_id"]: + tdata["date_original"] = text.parse_timestamp( + ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) return tdata @@ -400,9 +406,12 @@ class TwitterExtractor(Extractor): def _users_result(self, users): userfmt = self.config("users") - if not userfmt or userfmt == "timeline": - cls = TwitterTimelineExtractor + if not userfmt or userfmt == "user": + cls = TwitterUserExtractor fmt = (self.root + "/i/user/{rest_id}").format_map + elif userfmt == "timeline": + cls = TwitterTimelineExtractor + fmt = (self.root + "/id:{rest_id}/timeline").format_map elif userfmt == "media": cls = TwitterMediaExtractor fmt = (self.root + "/id:{rest_id}/media").format_map @@ -474,22 +483,7 @@ class TwitterUserExtractor(TwitterExtractor): subcategory = "user" pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" r"|i(?:/user/|ntent/user\?user_id=)(\d+))") - test = ( - ("https://twitter.com/supernaturepics", { - "options": (("include", "all"),), - "pattern": r"https://twitter\.com/supernaturepics" - r"/(photo|header_photo|timeline|tweets" - r"|media|with_replies|likes)$", - "count": 7, - }), - ("https://mobile.twitter.com/supernaturepics?p=i"), - ("https://www.twitter.com/id:2976459548"), - ("https://twitter.com/i/user/2976459548"), - ("https://twitter.com/intent/user?user_id=2976459548"), - ("https://fxtwitter.com/supernaturepics"), - ("https://vxtwitter.com/supernaturepics"), - ("https://x.com/supernaturepics"), - ) + example = "https://twitter.com/USER" def __init__(self, match): TwitterExtractor.__init__(self, match) @@ -517,22 +511,7 @@ class TwitterTimelineExtractor(TwitterExtractor): """Extractor for a Twitter user timeline""" subcategory = "timeline" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/timeline(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/timeline", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - # suspended account (#2216) - ("https://twitter.com/OptionalTypo/timeline", { - "exception": exception.NotFoundError, - }), - # suspended account user ID - ("https://twitter.com/id:772949683521978368/timeline", { - "exception": exception.NotFoundError, - }), - ("https://mobile.twitter.com/supernaturepics/timeline#t"), - ("https://www.twitter.com/id:2976459548/timeline"), - ) + example = "https://twitter.com/USER/timeline" def tweets(self): # yield initial batch of (media) tweets @@ -577,14 +556,7 @@ class TwitterTweetsExtractor(TwitterExtractor): """Extractor for Tweets from a user's Tweets timeline""" subcategory = "tweets" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/tweets(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/tweets", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/tweets#t"), - ("https://www.twitter.com/id:2976459548/tweets"), - ) + example = "https://twitter.com/USER/tweets" def tweets(self): return self.api.user_tweets(self.user) @@ -594,14 +566,7 @@ class TwitterRepliesExtractor(TwitterExtractor): """Extractor for Tweets from a user's timeline including replies""" subcategory = "replies" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/with_replies", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/with_replies#t"), - ("https://www.twitter.com/id:2976459548/with_replies"), - ) + example = "https://twitter.com/USER/with_replies" def tweets(self): return self.api.user_tweets_and_replies(self.user) @@ -611,14 +576,7 @@ class TwitterMediaExtractor(TwitterExtractor): """Extractor for Tweets from a user's Media timeline""" subcategory = "media" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/media", { - "range": "1-40", - "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - }), - ("https://mobile.twitter.com/supernaturepics/media#t"), - ("https://www.twitter.com/id:2976459548/media"), - ) + example = "https://twitter.com/USER/media" def tweets(self): return self.api.user_media(self.user) @@ -628,7 +586,7 @@ class TwitterLikesExtractor(TwitterExtractor): """Extractor for liked tweets""" subcategory = "likes" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/likes(?!\w)" - test = ("https://twitter.com/supernaturepics/likes",) + example = "https://twitter.com/USER/likes" def metadata(self): return {"user_likes": self.user} @@ -639,7 +597,7 @@ class TwitterLikesExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_liked"] = text.parse_timestamp( - (int(tweet["sortIndex"]) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata @@ -647,7 +605,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): """Extractor for bookmarked tweets""" subcategory = "bookmark" pattern = BASE_PATTERN + r"/i/bookmarks()" - test = ("https://twitter.com/i/bookmarks",) + example = "https://twitter.com/i/bookmarks" def tweets(self): return self.api.user_bookmarks() @@ -655,7 +613,7 @@ class TwitterBookmarkExtractor(TwitterExtractor): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_bookmarked"] = text.parse_timestamp( - (int(tweet["sortIndex"]) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000) return tdata @@ -663,11 +621,7 @@ class TwitterListExtractor(TwitterExtractor): """Extractor for Twitter lists""" subcategory = "list" pattern = BASE_PATTERN + r"/i/lists/(\d+)/?$" - test = ("https://twitter.com/i/lists/784214683683127296", { - "range": "1-40", - "count": 40, - "archive": False, - }) + example = "https://twitter.com/i/lists/12345" def tweets(self): return self.api.list_latest_tweets_timeline(self.user) @@ -677,11 +631,7 @@ class TwitterListMembersExtractor(TwitterExtractor): """Extractor for members of a Twitter list""" subcategory = "list-members" pattern = BASE_PATTERN + r"/i/lists/(\d+)/members" - test = ("https://twitter.com/i/lists/784214683683127296/members", { - "pattern": TwitterTimelineExtractor.pattern, - "range": "1-40", - "count": 40, - }) + example = "https://twitter.com/i/lists/12345/members" def items(self): self.login() @@ -692,10 +642,7 @@ class TwitterFollowingExtractor(TwitterExtractor): """Extractor for followed users""" subcategory = "following" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" - test = ( - ("https://twitter.com/supernaturepics/following"), - ("https://www.twitter.com/id:2976459548/following"), - ) + example = "https://twitter.com/USER/following" def items(self): self.login() @@ -706,11 +653,7 @@ class TwitterSearchExtractor(TwitterExtractor): """Extractor for Twitter search results""" subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" - test = ("https://twitter.com/search?q=nature", { - "range": "1-20", - "count": 20, - "archive": False, - }) + example = "https://twitter.com/search?q=QUERY" def metadata(self): return {"search": text.unquote(self.user)} @@ -741,10 +684,7 @@ class TwitterHashtagExtractor(TwitterExtractor): """Extractor for Twitter hashtags""" subcategory = "hashtag" pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" - test = ("https://twitter.com/hashtag/nature", { - "pattern": TwitterSearchExtractor.pattern, - "url": "3571c3a53b7647ea35517041fdc17f77ec5b2cb9", - }) + example = "https://twitter.com/hashtag/NAME" def items(self): url = "{}/search?q=%23{}".format(self.root, self.user) @@ -758,10 +698,7 @@ class TwitterEventExtractor(TwitterExtractor): directory_fmt = ("{category}", "Events", "{event[id]} {event[short_title]}") pattern = BASE_PATTERN + r"/i/events/(\d+)" - test = ("https://twitter.com/i/events/1484669206993903616", { - "range": "1-20", - "count": ">=1", - }) + example = "https://twitter.com/i/events/12345" def metadata(self): return {"event": self.api.live_event(self.user)} @@ -774,186 +711,7 @@ class TwitterTweetExtractor(TwitterExtractor): """Extractor for images from individual tweets""" subcategory = "tweet" pattern = BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" - test = ( - ("https://twitter.com/supernaturepics/status/604341487988576256", { - "url": "88a40f7d25529c2501c46f2218f9e0de9aa634b4", - "content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", - }), - # 4 images - ("https://twitter.com/perrypumas/status/894001459754180609", { - "url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", - }), - # video - ("https://twitter.com/perrypumas/status/1065692031626829824", { - "pattern": r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", - }), - # content with emoji, newlines, hashtags (#338) - ("https://twitter.com/playpokemon/status/1263832915173048321", { - "keyword": {"content": ( - r"re:Gear up for #PokemonSwordShieldEX with special Mystery " - "Gifts! \n\nYou’ll be able to receive four Galarian form " - "Pokémon with Hidden Abilities, plus some very useful items. " - "It’s our \\(Mystery\\) Gift to you, Trainers! \n\n❓🎁➡️ " - )}, - }), - # Reply to deleted tweet (#403, #838) - ("https://twitter.com/i/web/status/1170041925560258560", { - "pattern": r"https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", - }), - # 'replies' option (#705) - ("https://twitter.com/i/web/status/1170041925560258560", { - "options": (("replies", False),), - "count": 0, - }), - # 'replies' to self (#1254) - ("https://twitter.com/i/web/status/1424882930803908612", { - "options": (("replies", "self"),), - "count": 4, - "keyword": {"user": { - "description": "re:business email-- rhettaro.bloom@gmail.com " - "patreon- http://patreon.com/Princecanary", - "url": "http://princecanary.tumblr.com", - }}, - }), - ("https://twitter.com/i/web/status/1424898916156284928", { - "options": (("replies", "self"),), - "count": 1, - }), - # "quoted" option (#854) - ("https://twitter.com/StobiesGalaxy/status/1270755918330896395", { - "options": (("quoted", True),), - "pattern": r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", - "count": 8, - }), - # quoted tweet (#526, #854) - ("https://twitter.com/StobiesGalaxy/status/1270755918330896395", { - "pattern": r"https://pbs\.twimg\.com/media/EaK.+=jpg", - "count": 4, - }), - # different 'user' and 'author' in quoted Tweet (#3922) - ("https://twitter.com/web/status/1644907989109751810", { - "keyword": { - "author": {"id": 321629993 , "name": "Cakes_Comics"}, - "user" : {"id": 718928225360080897, "name": "StobiesGalaxy"}, - }, - }), - # TwitPic embeds (#579) - ("https://twitter.com/i/web/status/112900228289540096", { - "options": (("twitpic", True), ("cards", False)), - "pattern": r"https://\w+.cloudfront.net/photos/large/\d+.jpg", - "count": 2, # 1 duplicate - }), - # TwitPic URL not in 'urls' (#3792) - ("https://twitter.com/shimoigusaP/status/8138669971", { - "options": (("twitpic", True),), - "pattern": r"https://\w+.cloudfront.net/photos/large/\d+.png", - "count": 1, - }), - # Twitter card (#1005) - ("https://twitter.com/billboard/status/1306599586602135555", { - "options": (("cards", True),), - "pattern": r"https://pbs.twimg.com/card_img/\d+/", - }), - # unified_card image_website (#2875) - ("https://twitter.com/i/web/status/1561674543323910144", { - "options": (("cards", True),), - "pattern": r"https://pbs\.twimg\.com/media/F.+=jpg", - }), - # unified_card image_carousel_website - ("https://twitter.com/doax_vv_staff/status/1479438945662685184", { - "options": (("cards", True),), - "pattern": r"https://pbs\.twimg\.com/media/F.+=png", - "count": 6, - }), - # unified_card video_website (#2875) - ("https://twitter.com/bang_dream_1242/status/1561548715348746241", { - "options": (("cards", True),), - "pattern": r"https://video\.twimg\.com/amplify_video" - r"/1560607284333449216/vid/720x720/\w+\.mp4", - }), - # unified_card without type - ("https://twitter.com/i/web/status/1466183847628865544", { - "count": 0, - }), - # 'cards-blacklist' option - ("https://twitter.com/i/web/status/1571141912295243776", { - "options": (("cards", "ytdl"), - ("cards-blacklist", ("twitch.tv",))), - "count": 0, - }), - # retweet - ("https://twitter.com/jessica_3978/status/1296304589591810048", { - "options": (("retweets", True),), - "count": 2, - "keyword": { - "tweet_id" : 1296304589591810048, - "retweet_id": 1296296016002547713, - "date" : "dt:2020-08-20 04:34:32", - }, - }), - # original retweets (#1026) - ("https://twitter.com/jessica_3978/status/1296304589591810048", { - "options": (("retweets", "original"),), - "count": 2, - "keyword": { - "tweet_id" : 1296296016002547713, - "retweet_id": 1296296016002547713, - "date" : "dt:2020-08-20 04:00:28", - }, - }), - # all Tweets from a 'conversation' (#1319) - ("https://twitter.com/supernaturepics/status/604341487988576256", { - "options": (("conversations", True),), - "count": 5, - }), - # retweet with missing media entities (#1555) - ("https://twitter.com/morino_ya/status/1392763691599237121", { - "options": (("retweets", True),), - "count": 0, # private - }), - # deleted quote tweet (#2225) - ("https://twitter.com/i/web/status/1460044411165888515", { - "count": 0, - }), - # "Misleading" content - ("https://twitter.com/i/web/status/1486373748911575046", { - "count": 4, - }), - # age-restricted (#2354) - ("https://twitter.com/mightbecursed/status/1492954264909479936", { - "options": (("syndication", True),), - "keyword": {"date": "dt:2022-02-13 20:10:09"}, - "count": 1, - }), - # media alt texts / descriptions (#2617) - ("https://twitter.com/my0nruri/status/1528379296041299968", { - "keyword": {"description": "oc"} - }), - # '?format=...&name=...'-style URLs - ("https://twitter.com/poco_dandy/status/1150646424461176832", { - "options": (("cards", True),), - "pattern": r"https://pbs.twimg.com/card_img/157\d+/[\w-]+" - r"\?format=(jpg|png)&name=orig$", - "range": "1-2", - }), - # note tweet with long 'content' - ("https://twitter.com/i/web/status/1629193457112686592", { - "keyword": { - "content": """\ -BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have \ -just contradicted federal government regulators, saying that toxic air \ -pollutants in East Palestine, Ohio, could pose long-term risks. \n\nThe \ -Washington Post writes, "Three weeks after the toxic train derailment in \ -Ohio, an analysis of Environmental Protection Agency data has found nine air \ -pollutants at levels that could raise long-term health concerns in and around \ -East Palestine, according to an independent analysis. \n\n\"The analysis by \ -Texas A&M University seems to contradict statements by state and federal \ -regulators that air near the crash site is completely safe, despite residents \ -complaining about rashes, breathing problems and other health effects." \ -Your reaction.""", - }, - }), - ) + example = "https://twitter.com/USER/status/12345" def __init__(self, match): TwitterExtractor.__init__(self, match) @@ -1034,21 +792,7 @@ class TwitterAvatarExtractor(TwitterExtractor): filename_fmt = "avatar {date}.{extension}" archive_fmt = "AV_{user[id]}_{date}" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/photo" - test = ( - ("https://twitter.com/supernaturepics/photo", { - "pattern": r"https://pbs\.twimg\.com/profile_images" - r"/554585280938659841/FLVAlX18\.jpeg", - "keyword": { - "date": "dt:2015-01-12 10:26:49", - "extension": "jpeg", - "filename": "FLVAlX18", - "tweet_id": 554585280938659841, - }, - }), - ("https://twitter.com/User16/photo", { - "count": 0, - }), - ) + example = "https://twitter.com/USER/photo" def tweets(self): self.api._user_id_by_screen_name(self.user) @@ -1070,20 +814,7 @@ class TwitterBackgroundExtractor(TwitterExtractor): filename_fmt = "background {date}.{extension}" archive_fmt = "BG_{user[id]}_{date}" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/header_photo" - test = ( - ("https://twitter.com/supernaturepics/header_photo", { - "pattern": r"https://pbs\.twimg\.com/profile_banners" - r"/2976459548/1421058583", - "keyword": { - "date": "dt:2015-01-12 10:29:43", - "filename": "1421058583", - "tweet_id": 554586009367478272, - }, - }), - ("https://twitter.com/User16/header_photo", { - "count": 0, - }), - ) + example = "https://twitter.com/USER/header_photo" def tweets(self): self.api._user_id_by_screen_name(self.user) @@ -1103,13 +834,7 @@ class TwitterImageExtractor(Extractor): category = "twitter" subcategory = "image" pattern = r"https?://pbs\.twimg\.com/media/([\w-]+)(?:\?format=|\.)(\w+)" - test = ( - ("https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", { - "options": (("size", "4096x4096,orig"),), - "url": "cb3042a6f6826923da98f0d2b66c427e9385114c", - }), - ("https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig"), - ) + example = "https://pbs.twimg.com/media/ABCDE?format=jpg&name=orig" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index b298c27e..a1b87b9c 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -76,66 +76,7 @@ class UnsplashImageExtractor(UnsplashExtractor): """Extractor for a single unsplash photo""" subcategory = "image" pattern = BASE_PATTERN + r"/photos/([^/?#]+)" - test = ("https://unsplash.com/photos/lsoogGC_5dg", { - "pattern": r"https://images\.unsplash\.com/photo-1586348943529-" - r"beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", - "keyword": { - "alt_description": "re:silhouette of trees near body of water ", - "blur_hash": "LZP4uQS4jboe%#o0WCa}2doJNaaz", - "? categories": list, - "color": "#f3c08c", - "created_at": "2020-04-08T12:29:42Z", - "date": "dt:2020-04-08 12:29:42", - "description": "The Island", - "downloads": int, - "exif": { - "aperture": "11", - "exposure_time": "30", - "focal_length": "70.0", - "iso": 200, - "make": "Canon", - "model": "Canon EOS 5D Mark IV" - }, - "extension": "jpg", - "filename": "photo-1586348943529-beaae6c28db9", - "height": 6272, - "id": "lsoogGC_5dg", - "liked_by_user": False, - "likes": int, - "location": { - "city": "Beaver Dam", - "country": "United States", - "name": "Beaver Dam, WI 53916, USA", - "position": { - "latitude": 43.457769, - "longitude": -88.837329, - }, - }, - "promoted_at": "2020-04-08T15:12:03Z", - "sponsorship": None, - "tags": list, - "updated_at": str, - "user": { - "accepted_tos": True, - "bio": str, - "first_name": "Dave", - "id": "uMJXuywXLiU", - "instagram_username": "just_midwest_rock", - "last_name": "Hoefler", - "location": None, - "name": "Dave Hoefler", - "portfolio_url": None, - "total_collections": int, - "total_likes": int, - "total_photos": int, - "twitter_username": None, - "updated_at": str, - "username": "davehoefler", - }, - "views": int, - "width": 4480, - }, - }) + example = "https://unsplash.com/photos/ID" def photos(self): url = "{}/napi/photos/{}".format(self.root, self.item) @@ -146,12 +87,7 @@ class UnsplashUserExtractor(UnsplashExtractor): """Extractor for all photos of an unsplash user""" subcategory = "user" pattern = BASE_PATTERN + r"/@(\w+)/?$" - test = ("https://unsplash.com/@davehoefler", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/@USER" def photos(self): url = "{}/napi/users/{}/photos".format(self.root, self.item) @@ -163,12 +99,7 @@ class UnsplashFavoriteExtractor(UnsplashExtractor): """Extractor for all likes of an unsplash user""" subcategory = "favorite" pattern = BASE_PATTERN + r"/@(\w+)/likes" - test = ("https://unsplash.com/@davehoefler/likes", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/@USER/likes" def photos(self): url = "{}/napi/users/{}/likes".format(self.root, self.item) @@ -180,18 +111,7 @@ class UnsplashCollectionExtractor(UnsplashExtractor): """Extractor for an unsplash collection""" subcategory = "collection" pattern = BASE_PATTERN + r"/collections/([^/?#]+)(?:/([^/?#]+))?" - test = ( - ("https://unsplash.com/collections/3178572/winter", { - "pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "keyword": {"collection_id": "3178572", - "collection_title": "winter"}, - "range": "1-30", - "count": 30, - }), - ("https://unsplash.com/collections/3178572/"), - ("https://unsplash.com/collections/_8qJQ2bCMWE/2021.05"), - ) + example = "https://unsplash.com/collections/12345/TITLE" def __init__(self, match): UnsplashExtractor.__init__(self, match) @@ -210,13 +130,7 @@ class UnsplashSearchExtractor(UnsplashExtractor): """Extractor for unsplash search results""" subcategory = "search" pattern = BASE_PATTERN + r"/s/photos/([^/?#]+)(?:\?([^#]+))?" - test = ("https://unsplash.com/s/photos/hair-style", { - "pattern": r"https://(images|plus)\.unsplash\.com" - r"/((flagged/|premium_)?photo-\d+-\w+" - r"|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "range": "1-30", - "count": 30, - }) + example = "https://unsplash.com/s/photos/QUERY" def __init__(self, match): UnsplashExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/uploadir.py b/gallery_dl/extractor/uploadir.py index bd18c0ac..ce34e7de 100644 --- a/gallery_dl/extractor/uploadir.py +++ b/gallery_dl/extractor/uploadir.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,40 +20,7 @@ class UploadirFileExtractor(Extractor): filename_fmt = "{filename} ({id}).{extension}" archive_fmt = "{id}" pattern = r"(?:https?://)?uploadir\.com/(?:user/)?u(?:ploads)?/([^/?#]+)" - test = ( - # image - ("https://uploadir.com/u/rd3t46ry", { - "pattern": r"https://uploadir\.com/u/rd3t46ry", - "count": 1, - "keyword": { - "extension": "jpg", - "filename": "Chloe and Rachel 4K jpg", - "id": "rd3t46ry", - }, - }), - # archive - ("https://uploadir.com/uploads/gxe8ti9v/downloads/new", { - "pattern": r"https://uploadir\.com/uploads/gxe8ti9v/downloads", - "count": 1, - "keyword": { - "extension": "zip", - "filename": "NYAN-Mods-Pack#1", - "id": "gxe8ti9v", - }, - }), - # utf-8 filename - ("https://uploadir.com/u/fllda6xl", { - "pattern": r"https://uploadir\.com/u/fllda6xl", - "count": 1, - "keyword": { - "extension": "png", - "filename": "_圖片_🖼_image_", - "id": "fllda6xl", - }, - }), - ("https://uploadir.com/uploads/rd3t46ry"), - ("https://uploadir.com/user/uploads/rd3t46ry"), - ) + example = "https://uploadir.com/u/ID" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/urlshortener.py b/gallery_dl/extractor/urlshortener.py index 4b49a638..f2e65214 100644 --- a/gallery_dl/extractor/urlshortener.py +++ b/gallery_dl/extractor/urlshortener.py @@ -35,20 +35,7 @@ class UrlshortenerLinkExtractor(UrlshortenerExtractor): """Extractor for general-purpose URL shorteners""" subcategory = "link" pattern = BASE_PATTERN + r"/([^/?#]+)" - test = ( - ("https://bit.ly/3cWIUgq", { - "count": 1, - "pattern": "^https://gumroad.com/l/storm_b1", - }), - ("https://t.co/bCgBY8Iv5n", { - "count": 1, - "pattern": "^https://twitter.com/elonmusk/status/" - "1421395561324896257/photo/1", - }), - ("https://t.co/abcdefghij", { - "exception": exception.NotFoundError, - }), - ) + example = "https://bit.ly/abcde" def __init__(self, match): UrlshortenerExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index 6b1178e8..1ce969f7 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,10 +30,7 @@ class VanillarockPostExtractor(VanillarockExtractor): archive_fmt = "{filename}" pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?!category/|tag/)[^/?#]+)/?$") - test = ("https://vanilla-rock.com/mizuhashi_parsee-5", { - "url": "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", - "keyword": "b91df99b714e1958d9636748b1c81a07c3ef52c9", - }) + example = "https://vanilla-rock.com/TITLE" def items(self): extr = text.extract_from(self.request(self.root + self.path).text) @@ -66,18 +63,7 @@ class VanillarockTagExtractor(VanillarockExtractor): subcategory = "tag" pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?:tag|category)/[^?#]+)") - test = ( - ("https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", { - "pattern": VanillarockPostExtractor.pattern, - "count": ">= 12", - }), - (("https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83" - "%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95" - "%e3%83%88%e3%82%a8%e3%83%ad"), { - "pattern": VanillarockPostExtractor.pattern, - "count": ">= 5", - }), - ) + example = "https://vanilla-rock.com/tag/TAG" def items(self): url = self.root + self.path diff --git a/gallery_dl/extractor/vichan.py b/gallery_dl/extractor/vichan.py index 2fafb56e..79d7916e 100644 --- a/gallery_dl/extractor/vichan.py +++ b/gallery_dl/extractor/vichan.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -40,33 +40,7 @@ class VichanThreadExtractor(VichanExtractor): filename_fmt = "{time}{num:?-//} {filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" - test = ( - ("https://8kun.top/test/res/65248.html", { - "pattern": r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", - "count": ">= 8", - }), - # old-style file URLs (#1101) - # ("https://8kun.top/d/res/13258.html", { - # "pattern": r"https://media\.128ducks\.com/d/src/\d+(-\d)?\.\w+", - # "range": "1-20", - # }), - - ("https://wikieat.club/cel/res/25321.html", { - "pattern": r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+", - "count": ">= 200", - }), - - ("https://smuglo.li/a/res/1154380.html", { - "pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+", - "count": ">= 18", - "keyword": { - "board": "a", - "thread": "1154380", - "title": "Mob Psycho 100 Season 3", - }, - }), - ("https://smugloli.net/a/res/1145409.html"), - ) + example = "https://8kun.top/a/res/12345.html" def __init__(self, match): VichanExtractor.__init__(self, match) @@ -123,28 +97,7 @@ class VichanBoardExtractor(VichanExtractor): """Extractor for vichan boards""" subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" - test = ( - ("https://8kun.top/v/index.html", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://8kun.top/v/2.html"), - ("https://8kun.top/v/index.html?PageSpeed=noscript"), - - ("https://wikieat.club/cel/index.html", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://wikieat.club/cel/catalog.html"), - ("https://wikieat.club/cel/2.html"), - - ("https://smuglo.li/a", { - "pattern": VichanThreadExtractor.pattern, - "count": ">= 100", - }), - ("https://smuglo.li/a/1.html"), - ("https://smugloli.net/cute/catalog.html"), - ) + example = "https://8kun.top/a/" def __init__(self, match): VichanExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vipergirls.py b/gallery_dl/extractor/vipergirls.py index 084f9b25..4ee252ef 100644 --- a/gallery_dl/extractor/vipergirls.py +++ b/gallery_dl/extractor/vipergirls.py @@ -26,9 +26,6 @@ class VipergirlsExtractor(Extractor): cookies_domain = ".vipergirls.to" cookies_names = ("vg_userid", "vg_password") - def _init(self): - self.session.headers["Referer"] = self.root + "/" - def items(self): self.login() @@ -72,17 +69,7 @@ class VipergirlsThreadExtractor(VipergirlsExtractor): """Extractor for vipergirls threads""" subcategory = "thread" pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?$" - test = ( - (("https://vipergirls.to/threads/4328304" - "-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000"), { - "url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", - "count": 225, - }), - ("https://vipergirls.to/threads/6858916-Karina/page4", { - "count": 1279, - }), - ("https://vipergirls.to/threads/4328304"), - ) + example = "https://vipergirls.to/threads/12345-TITLE" def __init__(self, match): VipergirlsExtractor.__init__(self, match) @@ -103,21 +90,7 @@ class VipergirlsPostExtractor(VipergirlsExtractor): subcategory = "post" pattern = (BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)") - test = ( - (("https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-" - "Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081"), { - "pattern": r"https://vipr\.im/\w{12}$", - "range": "2-113", - "count": 112, - "keyword": { - "id": "116038081", - "imagecount": "113", - "number": "116038081", - "thread_id": "4328304", - "title": "FemJoy Danica - Simply Beautiful (x112) 3000x4500", - }, - }), - ) + example = "https://vipergirls.to/threads/12345-TITLE?p=23456#post23456" def __init__(self, match): VipergirlsExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 56924521..c9cd02f3 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -105,43 +105,7 @@ class VkPhotosExtractor(VkExtractor): pattern = (BASE_PATTERN + r"/(?:" r"(?:albums|photos|id)(-?\d+)" r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))") - test = ( - ("https://vk.com/id398982326", { - "pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1" - r"/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", - "count": ">= 35", - "keyword": { - "id": r"re:\d+", - "user": { - "id": "398982326", - "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", - "name": "", - "nick": "Dobrov Kurva", - }, - }, - }), - ("https://vk.com/cosplayinrussia", { - "range": "15-25", - "keyword": { - "id": r"re:\d+", - "user": { - "id" : "-165740836", - "info": str, - "name": "cosplayinrussia", - "nick": "Косплей | Cosplay 18+", - }, - }, - }), - # photos without width/height (#2535) - ("https://vk.com/id76957806", { - "pattern": r"https://sun\d+-\d+\.userapi\.com/", - "range": "1-9", - "count": 9, - }), - ("https://m.vk.com/albums398982326"), - ("https://www.vk.com/id398982326?profile=1"), - ("https://vk.com/albums-165740836"), - ) + example = "https://vk.com/id12345" def __init__(self, match): VkExtractor.__init__(self, match) @@ -181,18 +145,7 @@ class VkAlbumExtractor(VkExtractor): subcategory = "album" directory_fmt = ("{category}", "{user[id]}", "{album[id]}") pattern = BASE_PATTERN + r"/album(-?\d+)_(\d+)$" - test = ( - ("https://vk.com/album-165740836_281339889", { - "count": 12, - }), - # "Access denied" (#2556) - ("https://vk.com/album-53775183_00", { - "exception": exception.AuthorizationError, - }), - ("https://vk.com/album232175027_00", { - "exception": exception.AuthorizationError, - }), - ) + example = "https://vk.com/album12345_00" def __init__(self, match): VkExtractor.__init__(self, match) @@ -214,11 +167,7 @@ class VkTaggedExtractor(VkExtractor): subcategory = "tagged" directory_fmt = ("{category}", "{user[id]}", "tags") pattern = BASE_PATTERN + r"/tag(-?\d+)$" - test = ( - ("https://vk.com/tag304303884", { - "count": 44, - }), - ) + example = "https://vk.com/tag12345" def __init__(self, match): VkExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index 053a799f..14e3c7b4 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -116,16 +116,7 @@ class VscoUserExtractor(VscoExtractor): """Extractor for images from a user on vsco.co""" subcategory = "user" pattern = BASE_PATTERN + r"(?:/gallery|/images(?:/\d+)?)?/?(?:$|[?#])" - test = ( - ("https://vsco.co/missuri/gallery", { - "pattern": r"https://image(-aws.+)?\.vsco\.co" - r"/[0-9a-f/]+/[\w-]+\.\w+", - "range": "1-80", - "count": 80, - }), - ("https://vsco.co/missuri/images/1"), - ("https://vsco.co/missuri"), - ) + example = "https://vsco.co/USER/gallery" def images(self): url = "{}/{}/gallery".format(self.root, self.user) @@ -149,11 +140,7 @@ class VscoCollectionExtractor(VscoExtractor): directory_fmt = ("{category}", "{user}", "collection") archive_fmt = "c_{user}_{id}" pattern = BASE_PATTERN + r"/collection/" - test = ("https://vsco.co/vsco/collection/1", { - "pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", - "range": "1-80", - "count": 80, - }) + example = "https://vsco.co/USER/collection/12345" def images(self): url = "{}/{}/collection/1".format(self.root, self.user) @@ -176,29 +163,7 @@ class VscoImageExtractor(VscoExtractor): """Extractor for individual images on vsco.co""" subcategory = "image" pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)" - test = ( - ("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", { - "url": "a45f9712325b42742324b330c348b72477996031", - "content": "1394d070828d82078035f19a92f404557b56b83f", - "keyword": { - "id" : "5d34b93ef632433030707ce2", - "user" : "erenyildiz", - "grid" : "erenyildiz", - "meta" : dict, - "tags" : list, - "date" : "dt:2019-07-21 19:12:11", - "video" : False, - "width" : 1537, - "height": 1537, - "description": "re:Ni seviyorum. #vsco #vscox #vscochallenges", - }, - }), - ("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", { - "url": "08e7eef3301756ce81206c0b47c1e9373756a74a", - "content": "e739f058d726ee42c51c180a505747972a7dfa47", - "keyword": {"video" : True}, - }), - ) + example = "https://vsco.co/USER/media/0123456789abcdef" def __init__(self, match): VscoExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index 9e271098..479e8a85 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -57,15 +57,7 @@ class WallhavenSearchExtractor(WallhavenExtractor): directory_fmt = ("{category}", "{search[q]}") archive_fmt = "s_{search[q]}_{id}" pattern = r"(?:https?://)?wallhaven\.cc/search(?:/?\?([^#]+))?" - test = ( - ("https://wallhaven.cc/search?q=touhou"), - (("https://wallhaven.cc/search?q=id%3A87" - "&categories=111&purity=100&sorting=date_added&order=asc&page=3"), { - "pattern": (r"https://w\.wallhaven\.cc" - r"/full/\w\w/wallhaven-\w+\.\w+"), - "count": "<= 30", - }), - ) + example = "https://wallhaven.cc/search?q=QUERY" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -83,9 +75,7 @@ class WallhavenCollectionExtractor(WallhavenExtractor): subcategory = "collection" directory_fmt = ("{category}", "{username}", "{collection_id}") pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/favorites/(\d+)" - test = ("https://wallhaven.cc/user/AksumkA/favorites/74", { - "count": ">= 50", - }) + example = "https://wallhaven.cc/user/USER/favorites/12345" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -102,7 +92,7 @@ class WallhavenUserExtractor(WallhavenExtractor): """Extractor for a wallhaven user""" subcategory = "user" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/?$" - test = ("https://wallhaven.cc/user/AksumkA/",) + example = "https://wallhaven.cc/user/USER" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -123,10 +113,7 @@ class WallhavenCollectionsExtractor(WallhavenExtractor): """Extractor for all collections of a wallhaven user""" subcategory = "collections" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/favorites/?$" - test = ("https://wallhaven.cc/user/AksumkA/favorites", { - "pattern": WallhavenCollectionExtractor.pattern, - "count": 4, - }) + example = "https://wallhaven.cc/user/USER/favorites" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -146,12 +133,7 @@ class WallhavenUploadsExtractor(WallhavenExtractor): directory_fmt = ("{category}", "{username}") archive_fmt = "u_{username}_{id}" pattern = r"(?:https?://)?wallhaven\.cc/user/([^/?#]+)/uploads" - test = ("https://wallhaven.cc/user/AksumkA/uploads", { - "pattern": (r"https://[^.]+\.wallhaven\.cc" - r"/full/\w\w/wallhaven-\w+\.\w+"), - "range": "1-100", - "count": 100, - }) + example = "https://wallhaven.cc/user/USER/uploads" def __init__(self, match): WallhavenExtractor.__init__(self, match) @@ -170,41 +152,7 @@ class WallhavenImageExtractor(WallhavenExtractor): subcategory = "image" pattern = (r"(?:https?://)?(?:wallhaven\.cc/w/|whvn\.cc/" r"|w\.wallhaven\.cc/[a-z]+/\w\w/wallhaven-)(\w+)") - test = ( - ("https://wallhaven.cc/w/01w334", { - "pattern": (r"https://[^.]+\.wallhaven\.cc" - r"/full/01/wallhaven-01w334\.jpg"), - "content": "497212679383a465da1e35bd75873240435085a2", - "keyword": { - "id" : "01w334", - "width" : 1920, - "height" : 1200, - "resolution" : "1920x1200", - "ratio" : "1.6", - "colors" : list, - "tags" : list, - "file_size" : 278799, - "file_type" : "image/jpeg", - "purity" : "sfw", - "short_url" : "https://whvn.cc/01w334", - "source" : str, - "uploader" : { - "group" : "Owner/Developer", - "username" : "AksumkA", - }, - "date" : "dt:2014-08-31 06:17:19", - "wh_category": "anime", - "views" : int, - "favorites" : int, - }, - }), - # NSFW - ("https://wallhaven.cc/w/dge6v3", { - "url": "e4b802e70483f659d790ad5d0bd316245badf2ec", - }), - ("https://whvn.cc/01w334"), - ("https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg"), - ) + example = "https://wallhaven.cc/w/ID" def __init__(self, match): WallhavenExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wallpapercave.py b/gallery_dl/extractor/wallpapercave.py index 6c3af767..bce1026d 100644 --- a/gallery_dl/extractor/wallpapercave.py +++ b/gallery_dl/extractor/wallpapercave.py @@ -18,9 +18,7 @@ class WallpapercaveImageExtractor(Extractor): subcategory = "image" root = "https://wallpapercave.com" pattern = r"(?:https?://)?(?:www\.)?wallpapercave\.com" - test = ("https://wallpapercave.com/w/wp10270355", { - "content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", - }) + example = "https://wallpapercave.com/w/wp12345" def items(self): page = self.request(text.ensure_http_scheme(self.url)).text diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index bdedfcbc..6f152ed7 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,17 +21,7 @@ class WarosuThreadExtractor(Extractor): filename_fmt = "{tim}-{filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" pattern = r"(?:https?://)?(?:www\.)?warosu\.org/([^/]+)/thread/(\d+)" - test = ( - ("https://warosu.org/jp/thread/16656025", { - "url": "889d57246ed67e491e5b8f7f124e50ea7991e770", - "keyword": "c00ea4c5460c5986994f17bb8416826d42ca57c0", - }), - ("https://warosu.org/jp/thread/16658073", { - "url": "4500cf3184b067424fd9883249bd543c905fbecd", - "keyword": "7534edf4ec51891dbf44d775b73fbbefd52eec71", - "content": "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", - }), - ) + example = "https://warosu.org/a/thread/12345" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index c4d242a1..ddbfaa06 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -72,32 +72,7 @@ class WeasylExtractor(Extractor): class WeasylSubmissionExtractor(WeasylExtractor): subcategory = "submission" pattern = BASE_PATTERN + r"(?:~[\w~-]+/submissions|submission)/(\d+)" - test = ( - ("https://www.weasyl.com/~fiz/submissions/2031/a-wesley", { - "pattern": "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c29" - "40be928532785dfbf35c37622664d2fbb8114c3b063df969562fc5" - "1/fiz-a-wesley.png", - "keyword": { - "comments" : int, - "date" : "dt:2012-04-20 00:38:04", - "description" : "

(flex)

\n", - "favorites" : int, - "folder_name" : "Wesley Stuff", - "folderid" : 2081, - "friends_only": False, - "owner" : "Fiz", - "owner_login" : "fiz", - "rating" : "general", - "submitid" : 2031, - "subtype" : "visual", - "tags" : list, - "title" : "A Wesley!", - "type" : "submission", - "views" : int, - }, - }), - ("https://www.weasyl.com/submission/2031/a-wesley"), - ) + example = "https://www.weasyl.com/~USER/submissions/12345/TITLE" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -113,13 +88,7 @@ class WeasylSubmissionExtractor(WeasylExtractor): class WeasylSubmissionsExtractor(WeasylExtractor): subcategory = "submissions" pattern = BASE_PATTERN + r"(?:~|submissions/)([\w~-]+)/?$" - test = ( - ("https://www.weasyl.com/~tanidareal", { - "count": ">= 200" - }), - ("https://www.weasyl.com/submissions/tanidareal"), - ("https://www.weasyl.com/~aro~so") - ) + example = "https://www.weasyl.com/submissions/USER" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -134,9 +103,7 @@ class WeasylFolderExtractor(WeasylExtractor): subcategory = "folder" directory_fmt = ("{category}", "{owner_login}", "{folder_name}") pattern = BASE_PATTERN + r"submissions/([\w~-]+)\?folderid=(\d+)" - test = ("https://www.weasyl.com/submissions/tanidareal?folderid=7403", { - "count": ">= 12" - }) + example = "https://www.weasyl.com/submissions/USER?folderid=12345" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -157,14 +124,7 @@ class WeasylJournalExtractor(WeasylExtractor): filename_fmt = "{journalid} {title}.{extension}" archive_fmt = "{journalid}" pattern = BASE_PATTERN + r"journal/(\d+)" - test = ("https://www.weasyl.com/journal/17647/bbcode", { - "keyword": { - "title" : "BBCode", - "date" : "dt:2013-09-19 23:11:23", - "content": "

javascript:alert(42);

\n\n" - "

No more of that!

\n", - }, - }) + example = "https://www.weasyl.com/journal/12345" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -181,9 +141,7 @@ class WeasylJournalsExtractor(WeasylExtractor): filename_fmt = "{journalid} {title}.{extension}" archive_fmt = "{journalid}" pattern = BASE_PATTERN + r"journals/([\w~-]+)" - test = ("https://www.weasyl.com/journals/charmander", { - "count": ">= 2", - }) + example = "https://www.weasyl.com/journals/USER" def __init__(self, match): WeasylExtractor.__init__(self, match) @@ -203,9 +161,7 @@ class WeasylFavoriteExtractor(WeasylExtractor): subcategory = "favorite" directory_fmt = ("{category}", "{owner_login}", "Favorites") pattern = BASE_PATTERN + r"favorites\?userid=(\d+)" - test = ("https://www.weasyl.com/favorites?userid=184616&feature=submit", { - "count": ">= 5", - }) + example = "https://www.weasyl.com/favorites?userid=12345" def __init__(self, match): WeasylExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index b0384259..7e2b5ea3 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Mike Fährmann +# Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,36 +21,7 @@ class WebmshareVideoExtractor(Extractor): archive_fmt = "{id}" pattern = (r"(?:https?://)?(?:s\d+\.)?webmshare\.com" r"/(?:play/|download-webm/)?(\w{3,})") - test = ( - ("https://webmshare.com/O9mWY", { - "keyword": { - "date": "dt:2022-12-04 00:00:00", - "extension": "webm", - "filename": "O9mWY", - "height": 568, - "id": "O9mWY", - "thumb": "https://s1.webmshare.com/t/O9mWY.jpg", - "title": "Yeah buddy over here", - "url": "https://s1.webmshare.com/O9mWY.webm", - "views": int, - "width": 320, - }, - }), - ("https://s1.webmshare.com/zBGAg.webm", { - "keyword": { - "date": "dt:2018-12-07 00:00:00", - "height": 1080, - "id": "zBGAg", - "thumb": "https://s1.webmshare.com/t/zBGAg.jpg", - "title": "", - "url": "https://s1.webmshare.com/zBGAg.webm", - "views": int, - "width": 1920, - }, - }), - ("https://webmshare.com/play/zBGAg"), - ("https://webmshare.com/download-webm/zBGAg"), - ) + example = "https://webmshare.com/_ID_" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/webtoons.py b/gallery_dl/extractor/webtoons.py index 31592420..dc9a4f19 100644 --- a/gallery_dl/extractor/webtoons.py +++ b/gallery_dl/extractor/webtoons.py @@ -46,6 +46,8 @@ class WebtoonsEpisodeExtractor(WebtoonsBase, GalleryExtractor): archive_fmt = "{title_no}_{episode_no}_{num}" pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" r"/viewer(?:\?([^#'\"]+))") + example = ("https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" + "?title_no=123&episode_no=12345") test = ( (("https://www.webtoons.com/en/comedy/safely-endangered" "/ep-572-earth/viewer?title_no=352&episode_no=572"), { @@ -119,28 +121,7 @@ class WebtoonsComicExtractor(WebtoonsBase, Extractor): categorytransfer = True pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" r"/list(?:\?([^#]+))") - test = ( - # english - (("https://www.webtoons.com/en/comedy/live-with-yourself/" - "list?title_no=919"), { - "pattern": WebtoonsEpisodeExtractor.pattern, - "range": "1-15", - "count": ">= 15", - }), - # french - (("https://www.webtoons.com/fr/romance/subzero/" - "list?title_no=1845&page=3"), { - "count": ">= 15", - }), - # (#820) - (("https://www.webtoons.com/en/challenge/scoob-and-shag/" - "list?title_no=210827&page=9"), { - "count": ">= 18", - }), - # (#1643) - ("https://www.webtoons.com/es/romance/lore-olympus/" - "list?title_no=1725"), - ) + example = "https://www.webtoons.com/en/GENRE/TITLE/list?title_no=123" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index ae0fc4e3..168d5a0f 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -37,7 +37,6 @@ class WeiboExtractor(Extractor): cookies = _cookie_cache() if cookies is not None: self.cookies.update(cookies) - self.session.headers["Referer"] = self.root + "/" def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) @@ -219,16 +218,7 @@ class WeiboUserExtractor(WeiboExtractor): """Extractor for weibo user profiles""" subcategory = "user" pattern = USER_PATTERN + r"(?:$|#)" - test = ( - ("https://weibo.com/1758989602", { - "pattern": r"^https://weibo\.com/u/1758989602\?tabtype=feed$", - }), - ("https://weibo.com/u/1758989602"), - ("https://weibo.com/p/1758989602"), - ("https://m.weibo.cn/profile/2314621010"), - ("https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO"), - ("https://www.weibo.com/p/1003062314621010/home"), - ) + example = "https://weibo.com/USER" def initialize(self): pass @@ -248,10 +238,7 @@ class WeiboHomeExtractor(WeiboExtractor): """Extractor for weibo 'home' listings""" subcategory = "home" pattern = USER_PATTERN + r"\?tabtype=home" - test = ("https://weibo.com/1758989602?tabtype=home", { - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=home" def statuses(self): endpoint = "/profile/myhot" @@ -263,24 +250,7 @@ class WeiboFeedExtractor(WeiboExtractor): """Extractor for weibo user feeds""" subcategory = "feed" pattern = USER_PATTERN + r"\?tabtype=feed" - test = ( - ("https://weibo.com/1758989602?tabtype=feed", { - "range": "1-30", - "count": 30, - }), - ("https://weibo.com/zhouyuxi77?tabtype=feed", { - "keyword": {"status": {"user": {"id": 7488709788}}}, - "range": "1", - }), - ("https://www.weibo.com/n/周于希Sally?tabtype=feed", { - "keyword": {"status": {"user": {"id": 7488709788}}}, - "range": "1", - }), - # deleted (#2521) - ("https://weibo.com/u/7500315942?tabtype=feed", { - "count": 0, - }), - ) + example = "https://weibo.com/USER?tabtype=feed" def statuses(self): endpoint = "/statuses/mymblog" @@ -292,12 +262,7 @@ class WeiboVideosExtractor(WeiboExtractor): """Extractor for weibo 'video' listings""" subcategory = "videos" pattern = USER_PATTERN + r"\?tabtype=video" - test = ("https://weibo.com/1758989602?tabtype=video", { - "pattern": r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)" - r"/(../)?\w+\.mp4\?label=mp", - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=video" def statuses(self): endpoint = "/profile/getprofilevideolist" @@ -311,11 +276,7 @@ class WeiboNewvideoExtractor(WeiboExtractor): """Extractor for weibo 'newVideo' listings""" subcategory = "newvideo" pattern = USER_PATTERN + r"\?tabtype=newVideo" - test = ("https://weibo.com/1758989602?tabtype=newVideo", { - "pattern": r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", - "range": "1-30", - "count": 30, - }) + example = "https://weibo.com/USER?tabtype=newVideo" def statuses(self): endpoint = "/profile/getWaterFallContent" @@ -327,9 +288,7 @@ class WeiboArticleExtractor(WeiboExtractor): """Extractor for weibo 'article' listings""" subcategory = "article" pattern = USER_PATTERN + r"\?tabtype=article" - test = ("https://weibo.com/1758989602?tabtype=article", { - "count": 0, - }) + example = "https://weibo.com/USER?tabtype=article" def statuses(self): endpoint = "/statuses/mymblog" @@ -341,12 +300,7 @@ class WeiboAlbumExtractor(WeiboExtractor): """Extractor for weibo 'album' listings""" subcategory = "album" pattern = USER_PATTERN + r"\?tabtype=album" - test = ("https://weibo.com/1758989602?tabtype=album", { - "pattern": r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)" - r"|g\.us\.sinaimg\.cn/../\w+\.mp4)", - "range": "1-3", - "count": 3, - }) + example = "https://weibo.com/USER?tabtype=album" def statuses(self): endpoint = "/profile/getImageWall" @@ -368,57 +322,7 @@ class WeiboStatusExtractor(WeiboExtractor): """Extractor for images from a status on weibo.cn""" subcategory = "status" pattern = BASE_PATTERN + r"/(detail|status|\d+)/(\w+)" - test = ( - ("https://m.weibo.cn/detail/4323047042991618", { - "pattern": r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", - "keyword": {"status": { - "count": 1, - "date": "dt:2018-12-30 13:56:36", - }}, - }), - ("https://m.weibo.cn/detail/4339748116375525", { - "pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", - }), - # unavailable video (#427) - ("https://m.weibo.cn/status/4268682979207023", { - "exception": exception.NotFoundError, - }), - # non-numeric status ID (#664) - ("https://weibo.com/3314883543/Iy7fj4qVg"), - # original retweets (#1542) - ("https://m.weibo.cn/detail/4600272267522211", { - "options": (("retweets", "original"),), - "keyword": {"status": {"id": 4600167083287033}}, - }), - # type == livephoto (#2146) - ("https://weibo.com/5643044717/KkuDZ4jAA", { - "range": "2,4,6", - "pattern": r"https://video\.weibo\.com/media/play\?livephoto=" - r"https%3A%2F%2Fus.sinaimg.cn%2F\w+\.mov", - }), - # type == gif - ("https://weibo.com/1758989602/LvBhm5DiP", { - "pattern": r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104" - r"120005tc0E010\.mp4\?label=gif_mp4", - }), - # missing 'playback_list' (#2792) - ("https://weibo.com/2909128931/4409545658754086", { - "count": 10, - }), - # empty 'playback_list' (#3301) - ("https://weibo.com/1501933722/4142890299009993", { - "pattern": r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01" - r"000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori" - r"=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", - "count": 1, - }), - # mix_media_info (#3793) - ("https://weibo.com/2427303621/MxojLlLgQ", { - "count": 9, - }), - ("https://m.weibo.cn/status/4339748116375525"), - ("https://m.weibo.cn/5746766133/4339748116375525"), - ) + example = "https://weibo.com/detail/12345" def statuses(self): status = self._status_by_id(self.user) diff --git a/gallery_dl/extractor/wikiart.py b/gallery_dl/extractor/wikiart.py index 0e068588..938c0485 100644 --- a/gallery_dl/extractor/wikiart.py +++ b/gallery_dl/extractor/wikiart.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -69,10 +69,7 @@ class WikiartArtistExtractor(WikiartExtractor): subcategory = "artist" directory_fmt = ("{category}", "{artist[artistName]}") pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)/?$" - test = ("https://www.wikiart.org/en/thomas-cole", { - "url": "6844f207a5063c499fc1d5651b03127bc4fe2f73", - "keyword": "09230b5f504697119e267349bf92487e657a7384", - }) + example = "https://www.wikiart.org/en/ARTIST" def __init__(self, match): WikiartExtractor.__init__(self, match) @@ -94,16 +91,7 @@ class WikiartImageExtractor(WikiartArtistExtractor): """Extractor for individual paintings on wikiart.org""" subcategory = "image" pattern = BASE_PATTERN + r"/(?!(?:paintings|artists)-by-)([\w-]+)/([\w-]+)" - test = ( - ("https://www.wikiart.org/en/thomas-cole/the-departure-1838", { - "url": "976cc2545f308a650b5dbb35c29d3cee0f4673b3", - "keyword": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", - }), - # no year or '-' in slug - ("https://www.wikiart.org/en/huang-shen/summer", { - "url": "d7f60118c34067b2b37d9577e412dc1477b94207", - }), - ) + example = "https://www.wikiart.org/en/ARTIST/TITLE" def __init__(self, match): WikiartArtistExtractor.__init__(self, match) @@ -125,9 +113,7 @@ class WikiartArtworksExtractor(WikiartExtractor): subcategory = "artworks" directory_fmt = ("{category}", "Artworks by {group!c}", "{type}") pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)" - test = ("https://www.wikiart.org/en/paintings-by-media/grisaille", { - "url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", - }) + example = "https://www.wikiart.org/en/paintings-by-GROUP/TYPE" def __init__(self, match): WikiartExtractor.__init__(self, match) @@ -147,10 +133,7 @@ class WikiartArtistsExtractor(WikiartExtractor): """Extractor for artist collections on wikiart.org""" subcategory = "artists" pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)") - test = ("https://www.wikiart.org/en/artists-by-century/12", { - "pattern": WikiartArtistExtractor.pattern, - "count": ">= 8", - }) + example = "https://www.wikiart.org/en/artists-by-GROUP/TYPE" def __init__(self, match): WikiartExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index 5f02e94a..d3586c01 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -18,62 +18,7 @@ class WikifeetGalleryExtractor(GalleryExtractor): archive_fmt = "{type}_{celeb}_{pid}" pattern = (r"(?:https?://)(?:(?:www\.)?wikifeetx?|" r"men\.wikifeet)\.com/([^/?#]+)") - test = ( - ("https://www.wikifeet.com/Madison_Beer", { - "pattern": (r"https://pics\.wikifeet\.com/Madison_Beer" - r"-Feet-\d+\.jpg"), - "count" : ">= 352", - "keyword": { - "celeb" : "Madison_Beer", - "celebrity" : "Madison Beer", - "birthday" : "dt:1999-03-05 00:00:00", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "9 US", - "type" : "women", - "tags" : list, - }, - }), - ("https://www.wikifeetx.com/Tifa_Quinn", { - "pattern": (r"https://pics\.wikifeet\.com/Tifa_Quinn" - r"-Feet-\d+\.jpg"), - "count" : ">= 9", - "keyword": { - "celeb" : "Tifa_Quinn", - "celebrity" : "Tifa Quinn", - "birthday" : "[NOT SET]", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "4 US", - "type" : "women", - "tags" : list, - }, - }), - ("https://men.wikifeet.com/Chris_Hemsworth", { - "pattern": (r"https://pics\.wikifeet\.com/Chris_Hemsworth" - r"-Feet-\d+\.jpg"), - "count" : ">= 860", - "keyword": { - "celeb" : "Chris_Hemsworth", - "celebrity" : "Chris Hemsworth", - "birthday" : "dt:1983-08-11 00:00:00", - "birthplace": "Australia", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "12.5 US", - "type" : "men", - "tags" : list, - }, - }), - ) + example = "https://www.wikifeet.com/CELEB" def __init__(self, match): self.root = text.root_from_url(match.group(0)) diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index b308e74a..6dc9362a 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -32,54 +32,7 @@ class XhamsterGalleryExtractor(XhamsterExtractor): filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/photos/gallery/[^/?#]+)" - test = ( - ("https://xhamster.com/photos/gallery/11748968", { - "pattern": r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", - "count": ">= 144", - "keyword": { - "comments": int, - "count": int, - "favorite": bool, - "id": int, - "num": int, - "height": int, - "width": int, - "imageURL": str, - "pageURL": str, - "thumbURL": str, - "gallery": { - "date": "dt:2019-04-16 00:07:31", - "description": "", - "dislikes": int, - "id": 11748968, - "likes": int, - "tags": ["NON-Porn"], - "thumbnail": str, - "title": "Make the world better.", - "views": int, - }, - "user": { - "id": 16874672, - "name": "Anonymousrants", - "retired": bool, - "subscribers": int, - "url": "https://xhamster.com/users/anonymousrants", - "verified": bool, - }, - }, - }), - ("https://jp.xhamster2.com/photos/gallery/11748968", { - "pattern": r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", - "count": ">= 144", - }), - ("https://xhamster.com/photos/gallery/make-the-world-better-11748968"), - ("https://xhamster.com/photos/gallery/11748968"), - ("https://xhamster.one/photos/gallery/11748968"), - ("https://xhamster.desi/photos/gallery/11748968"), - ("https://xhamster2.com/photos/gallery/11748968"), - ("https://en.xhamster.com/photos/gallery/11748968"), - ("https://xhamster.porncache.net/photos/gallery/11748968"), - ) + example = "https://xhamster.com/photos/gallery/12345" def __init__(self, match): XhamsterExtractor.__init__(self, match) @@ -150,14 +103,7 @@ class XhamsterUserExtractor(XhamsterExtractor): """Extractor for all galleries of an xhamster user""" subcategory = "user" pattern = BASE_PATTERN + r"/users/([^/?#]+)(?:/photos)?/?(?:$|[?#])" - test = ( - ("https://xhamster.com/users/goldenpalomino/photos", { - "pattern": XhamsterGalleryExtractor.pattern, - "count": 50, - "range": "1-50", - }), - ("https://xhamster.com/users/nickname68"), - ) + example = "https://xhamster.com/users/USER/photos" def __init__(self, match): XhamsterExtractor.__init__(self, match) diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py index 46ea0743..a28d8f5a 100644 --- a/gallery_dl/extractor/xvideos.py +++ b/gallery_dl/extractor/xvideos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2019 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -28,30 +28,7 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): pattern = (r"(?:https?://)?(?:www\.)?xvideos\.com" r"/(?:profiles|amateur-channels|model-channels)" r"/([^/?#]+)/photos/(\d+)") - test = ( - ("https://www.xvideos.com/profiles/pervertedcouple/photos/751031", { - "count": 8, - "pattern": r"https://profile-pics-cdn\d+\.xvideos-cdn\.com" - r"/[^/]+\,\d+/videos/profiles/galleries/84/ca/37" - r"/pervertedcouple/gal751031/pic_\d+_big\.jpg", - "keyword": { - "gallery": { - "id" : 751031, - "title": "Random Stuff", - "tags" : list, - }, - "user": { - "id" : 20245371, - "name" : "pervertedcouple", - "display" : "Pervertedcouple", - "sex" : "Woman", - "description": str, - }, - }, - }), - ("https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12"), - ("https://www.xvideos.com/model-channels/pervertedcouple/photos/12"), - ) + example = "https://www.xvideos.com/profiles/USER/photos/12345" def __init__(self, match): self.user, self.gallery_id = match.groups() @@ -97,13 +74,7 @@ class XvideosUserExtractor(XvideosBase, Extractor): categorytransfer = True pattern = (r"(?:https?://)?(?:www\.)?xvideos\.com" r"/profiles/([^/?#]+)/?(?:#.*)?$") - test = ( - ("https://www.xvideos.com/profiles/pervertedcouple", { - "url": "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", - "keyword": "335a3304941ff2e666c0201e9122819b61b34adb", - }), - ("https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos"), - ) + example = "https://www.xvideos.com/profiles/USER" def __init__(self, match): Extractor.__init__(self, match) diff --git a/gallery_dl/extractor/ytdl.py b/gallery_dl/extractor/ytdl.py index 7f3c8de7..cb3c74c7 100644 --- a/gallery_dl/extractor/ytdl.py +++ b/gallery_dl/extractor/ytdl.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,7 @@ class YoutubeDLExtractor(Extractor): filename_fmt = "{title}-{id}.{extension}" archive_fmt = "{extractor_key} {id}" pattern = r"ytdl:(.*)" - test = ("ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9",) + example = "ytdl:https://www.youtube.com/watch?v=abcdefghijk" def __init__(self, match): # import main youtube_dl module diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index 8187db88..5fe1943c 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -111,23 +111,7 @@ class ZerochanTagExtractor(ZerochanExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") pattern = BASE_PATTERN + r"/(?!\d+$)([^/?#]+)/?(?:\?([^#]+))?" - test = ("https://www.zerochan.net/Perth+%28Kantai+Collection%29", { - "pattern": r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "count": "> 24", - "keyword": { - "extension": r"re:jpg|png", - "file_url": r"re:https://static\.zerochan\.net" - r"/.+\.full\.\d+\.(jpg|png)", - "filename": r"re:(Perth\.\(Kantai\.Collection\)" - r"|Kantai\.Collection)\.full\.\d+", - "height": r"re:^\d+$", - "id": r"re:^\d+$", - "name": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "search_tags": "Perth (Kantai Collection)", - "size": r"re:^\d+k$", - "width": r"re:^\d+$", - }, - }) + example = "https://www.zerochan.net/TAG" def __init__(self, match): ZerochanExtractor.__init__(self, match) @@ -177,40 +161,7 @@ class ZerochanTagExtractor(ZerochanExtractor): class ZerochanImageExtractor(ZerochanExtractor): subcategory = "image" pattern = BASE_PATTERN + r"/(\d+)" - test = ("https://www.zerochan.net/2920445", { - "pattern": r"https://static\.zerochan\.net/" - r"Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "keyword": { - "author": "YeFan 葉凡", - "date": "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net" - "/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height": 1366, - "id": 2920445, - "path": ["Kantai Collection", "Perth (Kantai Collection)"], - "size": 1975296, - "tags": [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Blonde Hair", - "Theme:Braids", - "Theme:Coat", - "Theme:Female", - "Theme:Firefighter Outfit", - "Theme:Group", - "Theme:Long Sleeves", - "Theme:Personification", - "Theme:Pins", - "Theme:Ribbon", - "Theme:Shirt", - "Theme:Short Hair", - ], - "uploader": "YukinoTokisaki", - "width": 1920, - }, - }) + example = "https://www.zerochan.net/12345" def __init__(self, match): ZerochanExtractor.__init__(self, match) diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 98ea186f..6098fc61 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -182,9 +182,10 @@ class StringFormatter(): if obj: break except Exception: - pass + obj = None else: - obj = self.default + if obj is None: + obj = self.default return fmt(obj) return wrap diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 459db5c7..e651d975 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -388,10 +388,25 @@ class DownloadJob(Job): try: if pextr.config("parent-skip"): job._skipcnt = self._skipcnt - self.status |= job.run() + status = job.run() self._skipcnt = job._skipcnt else: - self.status |= job.run() + status = job.run() + + if status: + self.status |= status + if "_fallback" in kwdict and self.fallback: + fallback = kwdict["_fallback"] = \ + iter(kwdict["_fallback"]) + try: + url = next(fallback) + except StopIteration: + pass + else: + text.nameext_from_url(url, kwdict) + if url.startswith("ytdl:"): + kwdict["extension"] = "" + self.handle_url(url, kwdict) break except exception.RestartExtraction: pass @@ -515,7 +530,7 @@ class DownloadJob(Job): elif skip == "terminate": self._skipexc = exception.TerminateExtraction elif skip == "exit": - self._skipexc = sys.exit + self._skipexc = SystemExit self._skipmax = text.parse_int(smax) else: # monkey-patch methods to always return False diff --git a/gallery_dl/postprocessor/compare.py b/gallery_dl/postprocessor/compare.py index 910e1d77..3bb63c80 100644 --- a/gallery_dl/postprocessor/compare.py +++ b/gallery_dl/postprocessor/compare.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2021 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,7 +10,6 @@ from .common import PostProcessor from .. import text, util, exception -import sys import os @@ -31,7 +30,7 @@ class ComparePP(PostProcessor): elif equal == "terminate": self._equal_exc = exception.TerminateExtraction elif equal == "exit": - self._equal_exc = sys.exit + self._equal_exc = SystemExit job.register_hooks({"file": ( self.enumerate diff --git a/gallery_dl/util.py b/gallery_dl/util.py index bf67a64d..62e7b4aa 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -239,8 +239,7 @@ def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): fp.write("\n") -def dump_response(response, fp, *, - headers=False, content=True, hide_auth=True): +def dump_response(response, fp, headers=False, content=True, hide_auth=True): """Write the contents of 'response' into a file-like object""" if headers: diff --git a/scripts/export_tests.py b/scripts/export_tests.py new file mode 100755 index 00000000..e95b0c2f --- /dev/null +++ b/scripts/export_tests.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2023 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import re +import sys +import itertools +import collections + +import util +from gallery_dl import extractor + + +FORMAT = '''\ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +{imports} + + +__tests__ = ( +{tests}\ +) +''' + + +def pprint(obj, indent=0, lmin=9, lmax=16): + + if isinstance(obj, str): + if obj.startswith("lit:"): + return f'''{obj[4:]}''' + + if "\\" in obj or obj.startswith("re:"): + prefix = "r" + else: + prefix = "" + + if "\n" in obj: + quote = '"""' + elif '"' in obj: + obj = re.sub(r'(?= 2 or not indent: + ws = " " * indent + + lkey = max(map(len, obj)) + if not indent: + if lkey < lmin: + lkey = lmin + elif lkey > lmax: + lkey = lmax + + lines = [] + lines.append("{") + for key, value in obj.items(): + if key.startswith("#blank-"): + lines.append("") + else: + lines.append( + f'''{ws} "{key}"''' + f'''{' '*(lkey - len(key))}: ''' + f'''{pprint(value, indent+4)},''' + ) + lines.append(f'''{ws}}}''') + return "\n".join(lines) + else: + key, value = obj.popitem() + return f'''{{"{key}": {pprint(value)}}}''' + + if isinstance(obj, list): + if not obj: + return "[]" + + if len(obj) >= 2: + ws = " " * indent + + lines = [] + lines.append("[") + lines.extend( + f'''{ws} {pprint(value, indent+4)},''' + for value in obj + ) + lines.append(f'''{ws}]''') + return "\n".join(lines) + else: + return f'''[{pprint(obj[0])}]''' + + if isinstance(obj, tuple): + if len(obj) == 1: + return f'''({pprint(obj[0], indent+4)},)''' + return f'''({", ".join(pprint(v, indent+4) for v in obj)})''' + + else: + return f'''{obj}''' + + +def extract_tests_from_source(lines): + tests = {} + + match_url = re.compile( + r''' (?:test = | )?\(\(?"([^"]+)"(.*)''').match + match_end = re.compile( + r" (\}\)| \}\),)\n$").match + first = 0 + url = "" + + for index, line in enumerate(lines): + if first and match_end(line): + tests[url] = lines[first-1:index+1] + first = 0 + + elif (m := match_url(line)): + offset = index + while not m[2]: + offset += 1 + next = lines[offset] + line = line[:-2] + next[next.index('"')+1:] + m = match_url(line) + url = m[1] + if m[2] in (",)", "),"): + tests[url] = lines[index-1:index+1] + first = 0 + else: + first = index + + return tests + + +def get_test_source(extr, *, cache={}): + try: + tests = cache[extr.__module__] + except KeyError: + path = sys.modules[extr.__module__].__file__ + with open(path) as fp: + lines = fp.readlines() + tests = cache[extr.__module__] = extract_tests_from_source(lines) + return tests.get(extr.url) or ("",) + return tests[extr.url] + + +def comment_from_source(source): + match = re.match(r"\s+#\s*(.+)", source[0]) + return match[1] if match else "" + + +def build_test(extr, data): + source = get_test_source(extr) + comment = comment_from_source(source) + + head = { + "#url" : extr.url, + "#comment" : comment.replace('"', "'"), + "#category": (extr.basecategory, + extr.category, + extr.subcategory), + "#class" : extr.__class__, + } + + if not comment: + del head["#comment"] + + instr = {} + + if not data: + data = {} + if (options := data.pop("options", None)): + instr["#options"] = { + name: value + for name, value in options + } + if (pattern := data.pop("pattern", None)): + if pattern in PATTERNS: + cls = PATTERNS[pattern] + pattern = f"lit:{pprint(cls)}.pattern" + instr["#pattern"] = pattern + if (exception := data.pop("exception", None)): + instr["#exception"] = exception + if (range := data.pop("range", None)): + instr["#range"] = range + if (count := data.pop("count", None)) is not None: + instr["#count"] = count + if (archive := data.pop("archive", None)) is not None: + instr["#archive"] = archive + if (extractor := data.pop("extractor", None)) is not None: + instr["#extractor"] = extractor + if (url := data.pop("url", None)): + instr["#sha1_url"] = url + if (metadata := data.pop("keyword", None)): + if isinstance(metadata, str) and len(metadata) == 40: + instr["#sha1_metadata"] = metadata + metadata = {} + if (content := data.pop("content", None)): + if isinstance(content, tuple): + content = list(content) + instr["#sha1_content"] = content + + if data: + print(extr) + for k in data: + print(k) + exit() + + return head, instr, metadata + + +def collect_patterns(): + return { + cls.pattern.pattern: cls + for cls in extractor._list_classes() + } + + +def collect_tests(whitelist=None): + tests = collections.defaultdict(list) + + for cls in extractor._list_classes(): + for url, data in cls._get_tests(): + + extr = cls.from_url(url) + if whitelist and extr.category not in whitelist: + continue + test = build_test(extr, data) + tests[extr.category].append(test) + + return tests + + +def export_tests(data): + imports = {} + tests = [] + + for head, instr, metadata in data: + + for v in itertools.chain( + head.values(), + instr.values() if instr else (), + metadata.values() if metadata else (), + ): + if not isinstance(v, type) or v.__module__ == "builtins": + continue + + module, _, name = v.__module__.rpartition(".") + if name[0].isdecimal(): + stmt = f'''\ +{module.partition(".")[0]} = __import__("{v.__module__}") +_{name} = getattr({module}, "{name}")''' + elif module: + stmt = f"from {module} import {name}" + else: + stmt = f"import {name}" + imports[v.__module__] = stmt + + test = pprint(head) + if instr: + test = f"{test[:-2]}{pprint(instr)[1:]}" + if metadata: + for k, v in metadata.items(): + if v == "type:datetime": + imports["datetime"] = "import datetime" + metadata[k] = "lit:datetime.datetime" + test = f"{test[:-1]}{pprint(metadata, lmin=0)[1:]}" + + tests.append(f"{test},\n\n") + + return FORMAT.format( + imports="\n".join(imports.values()), + tests="".join(tests), + ) + + +PATTERNS = None +DIRECTORY = "/tmp/_/results" + + +def main(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", "--target", + help="target directory", + ) + parser.add_argument( + "-c", "--category", action="append", + help="extractor categories to export", + ) + + args = parser.parse_args() + + if not args.target: + args.target = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "test", "results", + ) + + global PATTERNS + PATTERNS = collect_patterns() + + os.makedirs(args.target, exist_ok=True) + for name, tests in collect_tests(args.category).items(): + name = name.replace(".", "") + with util.lazy(f"{args.target}/{name}.py") as file: + file.write(export_tests(tests)) + + +if __name__ == "__main__": + main() diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index fe56e962..9f68d663 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -14,6 +14,11 @@ import collections import util from gallery_dl import extractor +try: + from test import results +except ImportError: + results = None + CATEGORY_MAP = { "2chan" : "Futaba Channel", @@ -153,7 +158,7 @@ SUBCATEGORY_MAP = { "tweets" : "", "user" : "User Profiles", "watch" : "Watches", - "following" : "", + "following" : "Followed Users", "related-pin" : "related Pins", "related-board": "", @@ -179,9 +184,6 @@ SUBCATEGORY_MAP = { "fapello": { "path": "Videos, Trending Posts, Popular Videos, Top Models", }, - "gfycat": { - "collections": "", - }, "hentaifoundry": { "story": "", }, @@ -225,6 +227,9 @@ SUBCATEGORY_MAP = { "sketch": "Sketch", "work": "individual Images", }, + "pornhub": { + "gifs": "", + }, "reddit": { "home": "Home Feed", }, @@ -312,7 +317,6 @@ AUTH_MAP = { "fanbox" : _COOKIES, "fantia" : _COOKIES, "flickr" : _OAUTH, - "gfycat" : "Supported", "furaffinity" : _COOKIES, "horne" : "Required", "idolcomplex" : "Supported", @@ -369,17 +373,8 @@ def domain(cls): if hasattr(cls, "root") and cls.root: return cls.root + "/" - if hasattr(cls, "https"): - scheme = "https" if cls.https else "http" - netloc = cls.__doc__.split()[-1] - return "{}://{}/".format(scheme, netloc) - - test = next(cls._get_tests(), None) - if test: - url = test[0] - return url[:url.find("/", 8)+1] - - return "" + url = cls.example + return url[:url.find("/", 8)+1] def category_text(c): @@ -434,14 +429,10 @@ def build_extractor_list(): for category, root in extr.instances: base[category].append(extr.subcategory) if category not in domains: - if not root: + if not root and results: # use domain from first matching test - for url, _ in extr._get_tests(): - if extr.from_url(url).category == category: - root = url[:url.index("/", 8)] - break - else: - continue + test = results.category(category)[0] + root = test["#class"].from_url(test["#url"]).root domains[category] = root + "/" # sort subcategory lists diff --git a/setup.cfg b/setup.cfg index 56d71087..e115e874 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -exclude = build,archive +exclude = .git,__pycache__,build,dist,archive,results ignore = E203,E226,W504 per-file-ignores = setup.py: E501 diff --git a/test/results/2chan.py b/test/results/2chan.py new file mode 100644 index 00000000..0a1cdd69 --- /dev/null +++ b/test/results/2chan.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.2chan") +_2chan = getattr(gallery_dl.extractor, "2chan") + + +__tests__ = ( +{ + "#url" : "https://dec.2chan.net/70/res/14565.htm", + "#category": ("", "2chan", "thread"), + "#class" : _2chan._2chanThreadExtractor, + "#pattern" : r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", + "#count" : ">= 3", + + "board" : "70", + "board_name": "新板提案", + "com" : str, + "fsize" : r"re:\d+", + "name" : "名無し", + "no" : r"re:1[45]\d\d\d", + "now" : r"re:22/../..\(.\)..:..:..", + "post" : "無題", + "server" : "dec", + "thread" : "14565", + "tim" : r"re:^\d{13}$", + "time" : r"re:^\d{10}$", + "title" : "ﾋﾛｱｶ板", +}, + +) diff --git a/test/results/2chen.py b/test/results/2chen.py new file mode 100644 index 00000000..589053fa --- /dev/null +++ b/test/results/2chen.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.2chen") +_2chen = getattr(gallery_dl.extractor, "2chen") +import datetime + + +__tests__ = ( +{ + "#url" : "https://sturdychan.help/tv/268929", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, + "#pattern" : r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", + "#count" : ">= 179", + + "board" : "tv", + "date" : datetime.datetime, + "hash" : r"re:[0-9a-f]{40}", + "name" : "Anonymous", + "no" : r"re:\d+", + "thread": "268929", + "time" : int, + "title" : "「/ttg/ #118: 🇧🇷 edition」", + "url" : str, +}, + +{ + "#url" : "https://2chen.club/tv/1", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, +}, + +{ + "#url" : "https://2chen.moe/jp/303786", + "#category": ("", "2chen", "thread"), + "#class" : _2chen._2chenThreadExtractor, +}, + +{ + "#url" : "https://sturdychan.help/co/", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, + "#pattern" : _2chen._2chenThreadExtractor.pattern, +}, + +{ + "#url" : "https://2chen.moe/co", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +{ + "#url" : "https://2chen.club/tv", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +{ + "#url" : "https://2chen.moe/co/catalog", + "#category": ("", "2chen", "board"), + "#class" : _2chen._2chenBoardExtractor, +}, + +) diff --git a/test/results/35photo.py b/test/results/35photo.py new file mode 100644 index 00000000..c0e29d53 --- /dev/null +++ b/test/results/35photo.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.35photo") +_35photo = getattr(gallery_dl.extractor, "35photo") + + +__tests__ = ( +{ + "#url" : "https://35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, + "#pattern" : r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", + "#count" : 9, +}, + +{ + "#url" : "https://35photo.pro/suhoveev", + "#comment" : "last photo ID (1267028) isn't given as 'photo-id=\"\" - " + "there are only 23 photos without the last one", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, + "#count" : ">= 33", +}, + +{ + "#url" : "https://en.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, +}, + +{ + "#url" : "https://ru.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class" : _35photo._35photoUserExtractor, +}, + +{ + "#url" : "https://35photo.pro/tags/landscape/", + "#category": ("", "35photo", "tag"), + "#class" : _35photo._35photoTagExtractor, + "#range" : "1-25", + "#count" : 25, + "#archive" : False, +}, + +{ + "#url" : "https://35photo.pro/genre_109/", + "#category": ("", "35photo", "genre"), + "#class" : _35photo._35photoGenreExtractor, +}, + +{ + "#url" : "https://35photo.pro/photo_753340/", + "#category": ("", "35photo", "image"), + "#class" : _35photo._35photoImageExtractor, + "#count" : 1, + + "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", + "id" : 753340, + "title" : "Winter walk", + "description": str, + "tags" : list, + "views" : int, + "favorites" : int, + "score" : int, + "type" : 0, + "date" : "15 авг, 2014", + "user" : "liya", + "user_id" : 20415, + "user_name" : "Liya Mirzaeva", +}, + +) diff --git a/test/results/3dbooru.py b/test/results/3dbooru.py new file mode 100644 index 00000000..d3d407bd --- /dev/null +++ b/test/results/3dbooru.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.3dbooru") +_3dbooru = getattr(gallery_dl.extractor, "3dbooru") + + +__tests__ = ( +{ + "#url" : "http://behoimi.org/post?tags=himekawa_azuru+dress", + "#category": ("booru", "3dbooru", "tag"), + "#class" : _3dbooru._3dbooruTagExtractor, + "#sha1_url" : "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", + "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", +}, + +{ + "#url" : "http://behoimi.org/pool/show/27", + "#category": ("booru", "3dbooru", "pool"), + "#class" : _3dbooru._3dbooruPoolExtractor, + "#sha1_url" : "da75d2d1475449d5ef0c266cb612683b110a30f2", + "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", +}, + +{ + "#url" : "http://behoimi.org/post/show/140852", + "#category": ("booru", "3dbooru", "post"), + "#class" : _3dbooru._3dbooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_url" : "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", + "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", + + "tags_character": "furude_rika", + "tags_copyright": "higurashi_no_naku_koro_ni", + "tags_model" : "himekawa_azuru", + "tags_general" : str, +}, + +{ + "#url" : "http://behoimi.org/post/popular_by_month?month=2&year=2013", + "#category": ("booru", "3dbooru", "popular"), + "#class" : _3dbooru._3dbooruPopularExtractor, + "#pattern" : r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", + "#count" : 20, +}, + +) diff --git a/test/results/4chan.py b/test/results/4chan.py new file mode 100644 index 00000000..c90e4107 --- /dev/null +++ b/test/results/4chan.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.4chan") +_4chan = getattr(gallery_dl.extractor, "4chan") + + +__tests__ = ( +{ + "#url" : "https://boards.4chan.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class" : _4chan._4chanThreadExtractor, + "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + "#sha1_content" : "20b7b51afa51c9c31a0020a0737b889532c8d7ec", +}, + +{ + "#url" : "https://boards.4channel.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class" : _4chan._4chanThreadExtractor, + "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", +}, + +{ + "#url" : "https://boards.4channel.org/po/", + "#category": ("", "4chan", "board"), + "#class" : _4chan._4chanBoardExtractor, + "#pattern" : _4chan._4chanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +) diff --git a/test/results/4chanarchives.py b/test/results/4chanarchives.py new file mode 100644 index 00000000..8aa8befd --- /dev/null +++ b/test/results/4chanarchives.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.4chanarchives") +_4chanarchives = getattr(gallery_dl.extractor, "4chanarchives") + + +__tests__ = ( +{ + "#url" : "https://4chanarchives.com/board/c/thread/2707110", + "#category": ("", "4chanarchives", "thread"), + "#class" : _4chanarchives._4chanarchivesThreadExtractor, + "#pattern" : r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", + "#count" : 2, + + "board" : "c", + "com" : str, + "name" : "Anonymous", + "no" : int, + "thread": "2707110", + "time" : r"re:2016-07-1\d \d\d:\d\d:\d\d", + "title" : "Ren Kagami from 'Oyako Neburi'", +}, + +{ + "#url" : "https://4chanarchives.com/board/c/", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, + "#pattern" : _4chanarchives._4chanarchivesThreadExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "https://4chanarchives.com/board/c", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, +}, + +{ + "#url" : "https://4chanarchives.com/board/c/10", + "#category": ("", "4chanarchives", "board"), + "#class" : _4chanarchives._4chanarchivesBoardExtractor, +}, + +) diff --git a/test/results/4plebs.py b/test/results/4plebs.py new file mode 100644 index 00000000..4f00027d --- /dev/null +++ b/test/results/4plebs.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archive.4plebs.org/tg/thread/54059290", + "#category": ("foolfuuka", "4plebs", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fd823f17b5001442b941fddcd9ec91bafedfbc79", +}, + +{ + "#url" : "https://archive.4plebs.org/tg/", + "#category": ("foolfuuka", "4plebs", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archive.4plebs.org/_/search/text/test/", + "#category": ("foolfuuka", "4plebs", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archive.4plebs.org/tg/gallery/1", + "#category": ("foolfuuka", "4plebs", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/500px.py b/test/results/500px.py new file mode 100644 index 00000000..39b618bc --- /dev/null +++ b/test/results/500px.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.500px") +_500px = getattr(gallery_dl.extractor, "500px") + + +__tests__ = ( +{ + "#url" : "https://500px.com/p/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, + "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", + "#range" : "1-99", + "#count" : 99, +}, + +{ + "#url" : "https://500px.com/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, +}, + +{ + "#url" : "https://web.500px.com/light_expression_photography", + "#category": ("", "500px", "user"), + "#class" : _500px._500pxUserExtractor, +}, + +{ + "#url" : "https://500px.com/p/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class" : _500px._500pxGalleryExtractor, + "#count" : 3, + "#sha1_url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", + + "gallery": dict, + "user" : dict, +}, + +{ + "#url" : "https://500px.com/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class" : _500px._500pxGalleryExtractor, +}, + +{ + "#url" : "https://500px.com/liked", + "#category": ("", "500px", "favorite"), + "#class" : _500px._500pxFavoriteExtractor, +}, + +{ + "#url" : "https://500px.com/photo/222049255/queen-of-coasts", + "#category": ("", "500px", "image"), + "#class" : _500px._500pxImageExtractor, + "#count" : 1, + "#sha1_url": "fbdf7df39325cae02f5688e9f92935b0e7113315", + + "camera" : "Canon EOS 600D", + "camera_info" : dict, + "comments" : list, + "comments_count" : int, + "created_at" : "2017-08-01T08:40:05+00:00", + "description" : str, + "editored_by" : None, + "editors_choice" : False, + "extension" : "jpg", + "feature" : "popular", + "feature_date" : "2017-08-01T09:58:28+00:00", + "focal_length" : "208", + "height" : 3111, + "id" : 222049255, + "image_format" : "jpg", + "image_url" : list, + "images" : list, + "iso" : "100", + "lens" : "EF-S55-250mm f/4-5.6 IS II", + "lens_info" : dict, + "liked" : None, + "location" : None, + "location_details": dict, + "name" : "Queen Of Coasts", + "nsfw" : False, + "privacy" : False, + "profile" : True, + "rating" : float, + "status" : 1, + "tags" : list, + "taken_at" : "2017-05-04T17:36:51+00:00", + "times_viewed" : int, + "url" : "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", + "user" : dict, + "user_id" : 12847235, + "votes_count" : int, + "watermark" : True, + "width" : 4637, +}, + +) diff --git a/test/results/8chan.py b/test/results/8chan.py new file mode 100644 index 00000000..43c08349 --- /dev/null +++ b/test/results/8chan.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.8chan") +_8chan = getattr(gallery_dl.extractor, "8chan") + + +__tests__ = ( +{ + "#url" : "https://8chan.moe/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, + "#pattern" : r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", + "#count" : 14, + + "archived" : False, + "autoSage" : False, + "boardDescription": "Film and Cinema", + "boardMarkdown" : None, + "boardName" : "Movies", + "boardUri" : "vhs", + "creation" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", + "cyclic" : False, + "email" : None, + "id" : r"re:^[0-9a-f]{6}$", + "locked" : False, + "markdown" : str, + "maxFileCount" : 5, + "maxFileSize" : "32.00 MB", + "maxMessageLength": 8001, + "message" : str, + "mime" : str, + "name" : "Anonymous", + "num" : int, + "originalName" : str, + "path" : r"re:/.media/[0-9a-f]{64}\.\w+$", + "pinned" : False, + "postId" : int, + "signedRole" : None, + "size" : int, + "threadId" : 4, + "thumb" : r"re:/.media/t_[0-9a-f]{64}$", + "uniquePosters" : 9, + "usesCustomCss" : True, + "usesCustomJs" : False, + "?wsPort" : 8880, + "?wssPort" : 2087, +}, + +{ + "#url" : "https://8chan.se/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, +}, + +{ + "#url" : "https://8chan.cc/vhs/res/4.html", + "#category": ("", "8chan", "thread"), + "#class" : _8chan._8chanThreadExtractor, +}, + +{ + "#url" : "https://8chan.moe/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +{ + "#url" : "https://8chan.moe/vhs/2.html", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, + "#pattern" : _8chan._8chanThreadExtractor.pattern, + "#count" : 23, +}, + +{ + "#url" : "https://8chan.se/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +{ + "#url" : "https://8chan.cc/vhs/", + "#category": ("", "8chan", "board"), + "#class" : _8chan._8chanBoardExtractor, +}, + +) diff --git a/test/results/8kun.py b/test/results/8kun.py new file mode 100644 index 00000000..53d16e13 --- /dev/null +++ b/test/results/8kun.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://8kun.top/test/res/65248.html", + "#category": ("vichan", "8kun", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", + "#count" : ">= 8", +}, + +{ + "#url" : "https://8kun.top/v/index.html", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://8kun.top/v/2.html", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://8kun.top/v/index.html?PageSpeed=noscript", + "#category": ("vichan", "8kun", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/8muses.py b/test/results/8muses.py new file mode 100644 index 00000000..da19e4ef --- /dev/null +++ b/test/results/8muses.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +gallery_dl = __import__("gallery_dl.extractor.8muses") +_8muses = getattr(gallery_dl.extractor, "8muses") + + +__tests__ = ( +{ + "#url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#pattern" : r"https://comics.8muses.com/image/fl/[\w-]+", + "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", + + "url" : str, + "hash" : str, + "page" : int, + "count": 6, + "album": { + "id" : 10467, + "title" : "Liar", + "path" : "Fakku Comics/mogg/Liar", + "parts" : [ + "Fakku Comics", + "mogg", + "Liar", + ], + "private": False, + "url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "parent" : 10464, + "views" : int, + "likes" : int, + "date" : "dt:2018-07-10 00:00:00", + }, +}, + +{ + "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/santa", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#pattern" : _8muses._8musesAlbumExtractor.pattern, + "#count" : ">= 3", + + "url" : str, + "name" : str, + "private": False, +}, + +{ + "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", + "#comment" : "custom sorting", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#count" : ">= 70", + + "name": r"re:^[R-Zr-z]", +}, + +{ + "#url" : "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", + "#comment" : "non-ASCII characters", + "#category": ("", "8muses", "album"), + "#class" : _8muses._8musesAlbumExtractor, + "#count" : 2, + + "name": r"re:From Trainers to Pokémons", +}, + +) diff --git a/test/results/94chan.py b/test/results/94chan.py new file mode 100644 index 00000000..6e792e30 --- /dev/null +++ b/test/results/94chan.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import jschan + + +__tests__ = ( +{ + "#url" : "https://94chan.org/art/thread/25.html", + "#category": ("jschan", "94chan", "thread"), + "#class" : jschan.JschanThreadExtractor, + "#pattern" : r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", + "#count" : ">= 15", +}, + +{ + "#url" : "https://94chan.org/art/", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, + "#pattern" : jschan.JschanThreadExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://94chan.org/art/2.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +{ + "#url" : "https://94chan.org/art/catalog.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +{ + "#url" : "https://94chan.org/art/index.html", + "#category": ("jschan", "94chan", "board"), + "#class" : jschan.JschanBoardExtractor, +}, + +) diff --git a/test/results/__init__.py b/test/results/__init__.py new file mode 100644 index 00000000..0fe87462 --- /dev/null +++ b/test/results/__init__.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import functools + +__directory__ = os.path.dirname(__file__) + + +@functools.lru_cache(maxsize=None) +def tests(name): + try: + module = __import__(name, globals(), None, (), 1) + return module.__tests__ + except Exception as exc: + print(exc) + return () + + +def all(): + ignore = ("__init__.py", "__pycache__") + for filename in os.listdir(__directory__): + if filename not in ignore: + yield from tests(filename[:-3]) + + +def category(category): + return tests(category.replace(".", "")) diff --git a/test/results/acidimg.py b/test/results/acidimg.py new file mode 100644 index 00000000..d61a7736 --- /dev/null +++ b/test/results/acidimg.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://acidimg.cc/img-5acb6b9de4640.html", + "#category": ("imagehost", "acidimg", "image"), + "#class" : imagehosts.AcidimgImageExtractor, + "#sha1_url" : "f132a630006e8d84f52d59555191ed82b3b64c04", + "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/adultempire.py b/test/results/adultempire.py new file mode 100644 index 00000000..1a2953ef --- /dev/null +++ b/test/results/adultempire.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import adultempire + + +__tests__ = ( +{ + "#url" : "https://www.adultempire.com/5998/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class" : adultempire.AdultempireGalleryExtractor, + "#range" : "1", + "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", + "#sha1_content" : "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", +}, + +{ + "#url" : "https://www.adultdvdempire.com/5683/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class" : adultempire.AdultempireGalleryExtractor, + "#sha1_url" : "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", + "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", +}, + +) diff --git a/test/results/aibooru.py b/test/results/aibooru.py new file mode 100644 index 00000000..78bd7273 --- /dev/null +++ b/test/results/aibooru.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://aibooru.online/posts?tags=center_frills&z=1", + "#category": ("Danbooru", "aibooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#pattern" : r"https://cdn\.aibooru\.online/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://safe.aibooru.online/posts?tags=center_frills", + "#category": ("Danbooru", "aibooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://aibooru.online/pools/1", + "#category": ("Danbooru", "aibooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, +}, + +{ + "#url" : "https://aibooru.online/posts/1", + "#category": ("Danbooru", "aibooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", +}, + +{ + "#url" : "https://aibooru.online/explore/posts/popular", + "#category": ("Danbooru", "aibooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/allgirlbooru.py b/test/results/allgirlbooru.py new file mode 100644 index 00000000..e1bfc11f --- /dev/null +++ b/test/results/allgirlbooru.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", + "#category": ("gelbooru_v01", "allgirlbooru", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", + "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", + "#category": ("gelbooru_v01", "allgirlbooru", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "b416800d2d2b072f80d3b37cfca9cb806fb25d51", + "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", + + "created_at": "2021-02-13 16:27:39", + "date" : "dt:2021-02-13 16:27:39", + "file_url" : "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", + "height" : "1200", + "id" : "107213", + "md5" : "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", + "rating" : "s", + "score" : str, + "source" : "", + "tags" : "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", + "uploader" : "Honochi31", + "width" : "1600", +}, + +) diff --git a/test/results/animereactor.py b/test/results/animereactor.py new file mode 100644 index 00000000..a83ac6ff --- /dev/null +++ b/test/results/animereactor.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://anime.reactor.cc/tag/Anime+Art", + "#category": ("reactor", "anime.reactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://anime.reactor.cc/user/Shuster", + "#category": ("reactor", "anime.reactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://anime.reactor.cc/post/3576250", + "#category": ("reactor", "anime.reactor", "post"), + "#class" : reactor.ReactorPostExtractor, +}, + +) diff --git a/test/results/architizer.py b/test/results/architizer.py new file mode 100644 index 00000000..3a751556 --- /dev/null +++ b/test/results/architizer.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import architizer + + +__tests__ = ( +{ + "#url" : "https://architizer.com/projects/house-lo/", + "#category": ("", "architizer", "project"), + "#class" : architizer.ArchitizerProjectExtractor, + "#pattern" : r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", + + "count" : 27, + "description": str, + "firm" : "Atelier Lina Bellovicova", + "gid" : "225496", + "location" : "Czechia", + "num" : int, + "size" : "1000 sqft - 3000 sqft", + "slug" : "house-lo", + "status" : "Built", + "subcategory": "project", + "title" : "House LO", + "type" : "Residential › Private House", + "year" : "2020", +}, + +{ + "#url" : "https://architizer.com/firms/olson-kundig/", + "#category": ("", "architizer", "firm"), + "#class" : architizer.ArchitizerFirmExtractor, + "#pattern" : architizer.ArchitizerProjectExtractor.pattern, + "#count" : ">= 90", +}, + +) diff --git a/test/results/archivedmoe.py b/test/results/archivedmoe.py new file mode 100644 index 00000000..90b48877 --- /dev/null +++ b/test/results/archivedmoe.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archived.moe/gd/thread/309639/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url" : "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", + "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", +}, + +{ + "#url" : "https://archived.moe/a/thread/159767162/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", +}, + +{ + "#url" : "https://archived.moe/gd/", + "#category": ("foolfuuka", "archivedmoe", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archived.moe/_/search/text/test/", + "#category": ("foolfuuka", "archivedmoe", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archived.moe/gd/gallery/2", + "#category": ("foolfuuka", "archivedmoe", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/archiveofsins.py b/test/results/archiveofsins.py new file mode 100644 index 00000000..3f3fcb93 --- /dev/null +++ b/test/results/archiveofsins.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archiveofsins.com/h/thread/4668813/", + "#category": ("foolfuuka", "archiveofsins", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url" : "f612d287087e10a228ef69517cf811539db9a102", + "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", +}, + +{ + "#url" : "https://archiveofsins.com/h/", + "#category": ("foolfuuka", "archiveofsins", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archiveofsins.com/h/gallery/3", + "#category": ("foolfuuka", "archiveofsins", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/artstation.py b/test/results/artstation.py new file mode 100644 index 00000000..262bad88 --- /dev/null +++ b/test/results/artstation.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import artstation +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.artstation.com/sungchoi/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, + "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/albums/all/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://sungchoi.artstation.com/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://sungchoi.artstation.com/projects/", + "#category": ("", "artstation", "user"), + "#class" : artstation.ArtstationUserExtractor, +}, + +{ + "#url" : "https://www.artstation.com/huimeiye/albums/770899", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.artstation.com/huimeiye/albums/770898", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://huimeiye.artstation.com/albums/770899", + "#category": ("", "artstation", "album"), + "#class" : artstation.ArtstationAlbumExtractor, +}, + +{ + "#url" : "https://www.artstation.com/mikf/likes", + "#category": ("", "artstation", "likes"), + "#class" : artstation.ArtstationLikesExtractor, + "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#count" : 6, +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/likes", + "#comment" : "no likes", + "#category": ("", "artstation", "likes"), + "#class" : artstation.ArtstationLikesExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.artstation.com/contests/thu-2017/challenges/20", + "#category": ("", "artstation", "challenge"), + "#class" : artstation.ArtstationChallengeExtractor, +}, + +{ + "#url" : "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", + "#category": ("", "artstation", "challenge"), + "#class" : artstation.ArtstationChallengeExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.artstation.com/search?query=ancient&sort_by=rank", + "#category": ("", "artstation", "search"), + "#class" : artstation.ArtstationSearchExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.artstation.com/artwork?sorting=latest", + "#category": ("", "artstation", "artwork"), + "#class" : artstation.ArtstationArtworkExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.artstation.com/artwork/LQVJr", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#pattern" : r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", + "#sha1_content": "7b113871465fdc09d127adfdc2767d51cf45a7e9", +}, + +{ + "#url" : "https://www.artstation.com/artwork/Db3dy", + "#comment" : "multiple images per project", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://www.artstation.com/artwork/g4WPK", + "#comment" : "embedded youtube video", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#options" : {"external": True}, + "#pattern" : "ytdl:https://www.youtube.com/embed/JNFfJtwwrU0", + "#range" : "2", +}, + +{ + "#url" : "https://www.artstation.com/artwork/3q3mXB", + "#comment" : "404 (#3016)", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://sungchoi.artstation.com/projects/LQVJr", + "#comment" : "alternate URL patterns", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, +}, + +{ + "#url" : "https://artstn.co/p/LQVJr", + "#category": ("", "artstation", "image"), + "#class" : artstation.ArtstationImageExtractor, +}, + +{ + "#url" : "https://www.artstation.com/sungchoi/following", + "#category": ("", "artstation", "following"), + "#class" : artstation.ArtstationFollowingExtractor, + "#pattern" : artstation.ArtstationUserExtractor.pattern, + "#count" : ">= 50", +}, + +) diff --git a/test/results/aryion.py b/test/results/aryion.py new file mode 100644 index 00000000..8b91e342 --- /dev/null +++ b/test/results/aryion.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import aryion + + +__tests__ = ( +{ + "#url" : "https://aryion.com/g4/gallery/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, + "#options" : {"recursive": False}, + "#pattern" : r"https://aryion\.com/g4/data\.php\?id=\d+$", + "#range" : "48-52", + "#count" : 5, +}, + +{ + "#url" : "https://aryion.com/g4/user/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, +}, + +{ + "#url" : "https://aryion.com/g4/latest.php?name=jameshoward", + "#category": ("", "aryion", "gallery"), + "#class" : aryion.AryionGalleryExtractor, +}, + +{ + "#url" : "https://aryion.com/g4/tags.php?tag=star+wars&p=19", + "#category": ("", "aryion", "tag"), + "#class" : aryion.AryionTagExtractor, + "#count" : ">= 5", +}, + +{ + "#url" : "https://aryion.com/g4/view/510079", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", + + "artist" : "jameshoward", + "user" : "jameshoward", + "filename" : "jameshoward-510079-subscribestar_150", + "extension" : "jpg", + "id" : 510079, + "width" : 1665, + "height" : 1619, + "size" : 784239, + "title" : "I'm on subscribestar now too!", + "description": r"re:Doesn't hurt to have a backup, right\?", + "tags" : [ + "Non-Vore", + "subscribestar", + ], + "date" : "dt:2019-02-16 19:30:34", + "path" : [], + "views" : int, + "favorites" : int, + "comments" : int, + "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", +}, + +{ + "#url" : "https://aryion.com/g4/view/588928", + "#comment" : "x-folder (#694)", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#pattern" : aryion.AryionPostExtractor.pattern, + "#count" : ">= 8", +}, + +{ + "#url" : "https://aryion.com/g4/view/537379", + "#comment" : "x-comic-folder (#945)", + "#category": ("", "aryion", "post"), + "#class" : aryion.AryionPostExtractor, + "#pattern" : aryion.AryionPostExtractor.pattern, + "#count" : 2, +}, + +) diff --git a/test/results/atfbooru.py b/test/results/atfbooru.py new file mode 100644 index 00000000..4bdd1b6d --- /dev/null +++ b/test/results/atfbooru.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://booru.allthefallen.moe/posts?tags=yume_shokunin", + "#category": ("Danbooru", "atfbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://booru.allthefallen.moe/pools/9", + "#category": ("Danbooru", "atfbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#count" : 6, + "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", +}, + +{ + "#url" : "https://booru.allthefallen.moe/posts/22", + "#category": ("Danbooru", "atfbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", +}, + +{ + "#url" : "https://booru.allthefallen.moe/explore/posts/popular", + "#category": ("Danbooru", "atfbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/b4k.py b/test/results/b4k.py new file mode 100644 index 00000000..5a2c13c2 --- /dev/null +++ b/test/results/b4k.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://arch.b4k.co/meta/thread/196/", + "#category": ("foolfuuka", "b4k", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", +}, + +{ + "#url" : "https://arch.b4k.co/meta/", + "#category": ("foolfuuka", "b4k", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://arch.b4k.co/meta/gallery/", + "#category": ("foolfuuka", "b4k", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/baraag.py b/test/results/baraag.py new file mode 100644 index 00000000..567d12c5 --- /dev/null +++ b/test/results/baraag.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://baraag.net/@pumpkinnsfw", + "#category": ("mastodon", "baraag", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://baraag.net/bookmarks", + "#category": ("mastodon", "baraag", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://baraag.net/users/pumpkinnsfw/following", + "#category": ("mastodon", "baraag", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://baraag.net/@pumpkinnsfw/104364170556898443", + "#category": ("mastodon", "baraag", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", +}, + +) diff --git a/test/results/bbc.py b/test/results/bbc.py new file mode 100644 index 00000000..83b4923c --- /dev/null +++ b/test/results/bbc.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import bbc + + +__tests__ = ( +{ + "#url" : "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", + "#category": ("", "bbc", "gallery"), + "#class" : bbc.BbcGalleryExtractor, + "#pattern" : r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", + "#count" : 37, + + "programme": "p084qtzs", + "path" : [ + "BBC One", + "Doctor Who", + "The Timeless Children", + ], +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/p084qtzs", + "#category": ("", "bbc", "gallery"), + "#class" : bbc.BbcGalleryExtractor, +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries", + "#category": ("", "bbc", "programme"), + "#class" : bbc.BbcProgrammeExtractor, + "#pattern" : bbc.BbcGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", + "#category": ("", "bbc", "programme"), + "#class" : bbc.BbcProgrammeExtractor, + "#pattern" : bbc.BbcGalleryExtractor.pattern, + "#count" : ">= 100", +}, + +) diff --git a/test/results/bbw-chan.py b/test/results/bbw-chan.py new file mode 100644 index 00000000..74913ff9 --- /dev/null +++ b/test/results/bbw-chan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://bbw-chan.nl/bbwdraw/res/499.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://bbw-chan\.nl/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count" : ">= 352", +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/res/489.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 148", +}, + +{ + "#url" : "https://bbw-chan.nl/bbwdraw/2.html", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/bcbnsfw.py b/test/results/bcbnsfw.py new file mode 100644 index 00000000..8d35e36e --- /dev/null +++ b/test/results/bcbnsfw.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import szurubooru + + +__tests__ = ( +{ + "#url" : "https://booru.bcbnsfw.space/posts/query=simple_background", + "#category": ("szurubooru", "bcbnsfw", "tag"), + "#class" : szurubooru.SzurubooruTagExtractor, +}, + +{ + "#url" : "https://booru.bcbnsfw.space/post/1599", + "#category": ("szurubooru", "bcbnsfw", "post"), + "#class" : szurubooru.SzurubooruPostExtractor, + "#pattern" : r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", + "#sha1_content": "0c38fc612ba1f03950fad31c4f80a1fccdab1096", +}, + +) diff --git a/test/results/behance.py b/test/results/behance.py new file mode 100644 index 00000000..7b39b5b4 --- /dev/null +++ b/test/results/behance.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import behance +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.behance.net/gallery/17386197/A-Short-Story", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#count" : 2, + "#sha1_url": "ab79bd3bef8d3ae48e6ac74fd995c1dfaec1b7d2", + + "id" : 17386197, + "name" : r"re:\"Hi\". A short story about the important things ", + "owners": [ + "Place Studio", + "Julio César Velazquez", + ], + "fields": [ + "Animation", + "Character Design", + "Directing", + ], + "tags" : list, + "module": dict, + "date" : "dt:2014-06-03 15:41:51", +}, + +{ + "#url" : "https://www.behance.net/gallery/21324767/Nevada-City", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#count" : 6, + "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", + + "owners": ["Alex Strohl"], +}, + +{ + "#url" : "https://www.behance.net/gallery/88276087/Audi-R8-RWD", + "#comment" : "'media_collection' modules", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#pattern" : r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", + "#count" : 20, + "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", +}, + +{ + "#url" : "https://www.behance.net/gallery/101185577/COLCCI", + "#comment" : "'video' modules (#1282)", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#pattern" : r"https://cdn-prod-ccv\.adobe\.com/\w+/rend/\w+_720\.mp4\?", + "#count" : 3, +}, + +{ + "#url" : "https://www.behance.net/gallery/177464639/Kimori", + "#comment" : "mature content (#4417)", + "#category": ("", "behance", "gallery"), + "#class" : behance.BehanceGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://www.behance.net/alexstrohl", + "#category": ("", "behance", "user"), + "#class" : behance.BehanceUserExtractor, + "#pattern" : behance.BehanceGalleryExtractor.pattern, + "#count" : ">= 11", +}, + +{ + "#url" : "https://www.behance.net/collection/71340149/inspiration", + "#category": ("", "behance", "collection"), + "#class" : behance.BehanceCollectionExtractor, + "#pattern" : behance.BehanceGalleryExtractor.pattern, + "#count" : ">= 150", +}, + +) diff --git a/test/results/bitly.py b/test/results/bitly.py new file mode 100644 index 00000000..cbfcf0eb --- /dev/null +++ b/test/results/bitly.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import urlshortener + + +__tests__ = ( +{ + "#url" : "https://bit.ly/3cWIUgq", + "#category": ("urlshortener", "bitly", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#pattern" : "^https://gumroad.com/l/storm_b1", + "#count" : 1, +}, + +) diff --git a/test/results/blogger.py b/test/results/blogger.py new file mode 100644 index 00000000..214d450d --- /dev/null +++ b/test/results/blogger.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import blogger + + +__tests__ = ( +{ + "#url" : "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : "https://3.bp.blogspot.com/.*/s0/Icy-Moonrise-.*.jpg", + "#sha1_url": "9928429fb62f712eb4de80f53625eccecc614aae", + + "blog": { + "date" : "dt:2010-11-21 18:19:42", + "description": "", + "id" : "5623928067739466034", + "kind" : "blogger#blog", + "locale" : dict, + "name" : "Julian Bunker Photography", + "pages" : int, + "posts" : int, + "published" : "2010-11-21T10:19:42-08:00", + "updated" : str, + "url" : "http://julianbphotography.blogspot.com/", + }, + "post": { + "author" : "Julian Bunker", + "content" : str, + "date" : "dt:2010-12-26 01:08:00", + "etag" : str, + "id" : "6955139236418998998", + "kind" : "blogger#post", + "published": "2010-12-25T17:08:00-08:00", + "replies" : "0", + "title" : "Moon Rise", + "updated" : "2011-12-06T05:21:24-08:00", + "url" : r"re:.+/2010/12/moon-rise.html$", + }, + "num" : int, + "url" : str, +}, + +{ + "#url" : "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, +}, + +{ + "#url" : "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", + "#comment" : "video (#587)", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : r"https://.+\.googlevideo\.com/videoplayback", +}, + +{ + "#url" : "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", + "#comment" : "new image domain (#2204)", + "#category": ("", "blogger", "post"), + "#class" : blogger.BloggerPostExtractor, + "#pattern" : "https://blogger.googleusercontent.com/img/a/.+=s0$", + "#count" : 8, +}, + +{ + "#url" : "https://julianbphotography.blogspot.com/", + "#category": ("", "blogger", "blog"), + "#class" : blogger.BloggerBlogExtractor, + "#pattern" : r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg", + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "blogger:https://www.kefblog.com.ng/", + "#category": ("", "blogger", "blog"), + "#class" : blogger.BloggerBlogExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://julianbphotography.blogspot.com/search?q=400mm", + "#category": ("", "blogger", "search"), + "#class" : blogger.BloggerSearchExtractor, + "#count" : "< 10", + + "query": "400mm", +}, + +{ + "#url" : "https://dmmagazine.blogspot.com/search/label/D%26D", + "#category": ("", "blogger", "label"), + "#class" : blogger.BloggerLabelExtractor, + "#range" : "1-25", + "#count" : 25, + + "label": "D&D", +}, + +) diff --git a/test/results/booruvar.py b/test/results/booruvar.py new file mode 100644 index 00000000..8beb45e1 --- /dev/null +++ b/test/results/booruvar.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://booru.borvar.art/posts?tags=chibi&z=1", + "#category": ("Danbooru", "booruvar", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#pattern" : r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://booru.borvar.art/pools/2", + "#category": ("Danbooru", "booruvar", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#count" : 4, + "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", +}, + +{ + "#url" : "https://booru.borvar.art/posts/1487", + "#category": ("Danbooru", "booruvar", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", +}, + +{ + "#url" : "https://booru.borvar.art/explore/posts/popular", + "#category": ("Danbooru", "booruvar", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +) diff --git a/test/results/bunkr.py b/test/results/bunkr.py new file mode 100644 index 00000000..5a66d02c --- /dev/null +++ b/test/results/bunkr.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import bunkr + + +__tests__ = ( +{ + "#url" : "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://cdn\.bunkr\.ru/test-テスト-\"&>-QjgneIQv\.png", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "album_id" : "Lktg9Keq", + "album_name": "test テスト \"&>", + "count" : 1, + "filename" : "test-テスト-\"&>-QjgneIQv", + "id" : "QjgneIQv", + "name" : "test-テスト-\"&>", + "num" : int, +}, + +{ + "#url" : "https://app.bunkr.ru/a/ptRHaCn2", + "#comment" : "mp4 (#2239)", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://media-files\.bunkr\.ru/_-RnHoW69L\.mp4", + "#sha1_content": "80e61d1dbc5896ae7ef9a28734c747b28b320471", +}, + +{ + "#url" : "https://bunkr.is/a/iXTTc1o2", + "#comment" : "cdn4", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://(cdn|media-files)4\.bunkr\.ru/", + "#sha1_content": "da29aae371b7adc8c5ef8e6991b66b69823791e8", + + "album_id" : "iXTTc1o2", + "album_name" : "test2", + "album_size" : "691.1 KB", + "count" : 2, + "description": "072022", + "filename" : r"re:video-wFO9FtxG|image-sZrQUeOx", + "id" : r"re:wFO9FtxG|sZrQUeOx", + "name" : r"re:video|image", + "num" : int, +}, + +{ + "#url" : "https://bunkrr.su/a/j1G29CnD", + "#comment" : "cdn12 .ru TLD (#4147)", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, + "#pattern" : r"https://(cdn12.bunkr.ru|media-files12.bunkr.la)/\w+", + "#count" : 8, +}, + +{ + "#url" : "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.la/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.is/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +{ + "#url" : "https://bunkr.to/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class" : bunkr.BunkrAlbumExtractor, +}, + +) diff --git a/test/results/catbox.py b/test/results/catbox.py new file mode 100644 index 00000000..b977a52e --- /dev/null +++ b/test/results/catbox.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import catbox + + +__tests__ = ( +{ + "#url" : "https://catbox.moe/c/1igcbe", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, + "#pattern" : r"https://files\.catbox\.moe/\w+\.\w{3}$", + "#count" : 3, + "#sha1_url" : "35866a88c29462814f103bc22ec031eaeb380f8a", + "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", + + "album_id" : "1igcbe", + "album_name" : "test", + "date" : "dt:2022-08-18 00:00:00", + "description": "album test &>", +}, + +{ + "#url" : "https://www.catbox.moe/c/cd90s1", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, +}, + +{ + "#url" : "https://catbox.moe/c/w7tm47#", + "#category": ("", "catbox", "album"), + "#class" : catbox.CatboxAlbumExtractor, +}, + +{ + "#url" : "https://files.catbox.moe/8ih3y7.png", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, + "#pattern" : r"^https://files\.catbox\.moe/8ih3y7\.png$", + "#count" : 1, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://litter.catbox.moe/t8v3n9.png", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, +}, + +{ + "#url" : "https://de.catbox.moe/bjdmz1.jpg", + "#category": ("", "catbox", "file"), + "#class" : catbox.CatboxFileExtractor, +}, + +) diff --git a/test/results/cavemanon.py b/test/results/cavemanon.py new file mode 100644 index 00000000..4fc0af3c --- /dev/null +++ b/test/results/cavemanon.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://booru.cavemanon.xyz/post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, +}, + +{ + "#url" : "https://booru.cavemanon.xyz/index.php?q=post/view/8335", + "#category": ("shimmie2", "cavemanon", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", + "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", + + "extension": "png", + "file_url" : "https://booru.cavemanon.xyz/index.php?q=image/8335.png", + "filename" : "8335", + "height" : 460, + "id" : 8335, + "md5" : "", + "size" : 0, + "tags" : "Color Fang Food Pterodactyl discord_emote transparent", + "width" : 459, +}, + +) diff --git a/test/results/chelseacrew.py b/test/results/chelseacrew.py new file mode 100644 index 00000000..7c12613f --- /dev/null +++ b/test/results/chelseacrew.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://chelseacrew.com/collections/flats", + "#category": ("shopify", "chelseacrew", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://chelseacrew.com/collections/flats/products/dora", + "#category": ("shopify", "chelseacrew", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/comicvine.py b/test/results/comicvine.py new file mode 100644 index 00000000..fa77c491 --- /dev/null +++ b/test/results/comicvine.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import comicvine + + +__tests__ = ( +{ + "#url" : "https://comicvine.gamespot.com/jock/4040-5653/images/", + "#category": ("", "comicvine", "tag"), + "#class" : comicvine.ComicvineTagExtractor, + "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", + "#count" : ">= 140", +}, + +{ + "#url" : "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", + "#category": ("", "comicvine", "tag"), + "#class" : comicvine.ComicvineTagExtractor, + "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", + "#count" : ">= 450", +}, + +) diff --git a/test/results/coomerparty.py b/test/results/coomerparty.py new file mode 100644 index 00000000..dfc4a188 --- /dev/null +++ b/test/results/coomerparty.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kemonoparty + + +__tests__ = ( +{ + "#url" : "https://coomer.party/onlyfans/user/alinity/post/125962203", + "#comment" : "coomer.party (#2100)", + "#category": ("", "coomerparty", "onlyfans"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg", +}, + +) diff --git a/test/results/cyberdrop.py b/test/results/cyberdrop.py new file mode 100644 index 00000000..b163ca47 --- /dev/null +++ b/test/results/cyberdrop.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import cyberdrop + + +__tests__ = ( +{ + "#url" : "https://cyberdrop.me/a/keKRjm4t", + "#comment" : "images", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class" : cyberdrop.CyberdropAlbumExtractor, + "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$", + + "album_id" : "keKRjm4t", + "album_name" : "Fate (SFW)", + "album_size" : 150069254, + "count" : 62, + "date" : "dt:2020-06-18 13:14:20", + "description": "", + "id" : r"re:\w{8}", +}, + +{ + "#url" : "https://cyberdrop.to/a/l8gIAXVD", + "#comment" : "videos", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class" : cyberdrop.CyberdropAlbumExtractor, + "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$", + "#count" : 31, + + "album_id" : "l8gIAXVD", + "album_name": "Achelois17 videos", + "album_size": 652037121, + "date" : "dt:2020-06-16 15:40:44", +}, + +) diff --git a/test/results/danbooru.py b/test/results/danbooru.py new file mode 100644 index 00000000..c64c693e --- /dev/null +++ b/test/results/danbooru.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import danbooru + + +__tests__ = ( +{ + "#url" : "https://danbooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts?tags=mushishi", + "#comment" : "test page transitions", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#count" : ">= 300", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", + "#comment" : "'external' option (#1747)", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, + "#options" : {"external": True}, + "#pattern" : r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", +}, + +{ + "#url" : "https://hijiribe.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://sonohara.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://safebooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class" : danbooru.DanbooruTagExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/pools/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", +}, + +{ + "#url" : "https://danbooru.donmai.us/pool/show/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/posts/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + + "date": "dt:2008-08-12 04:46:05", +}, + +{ + "#url" : "https://danbooru.donmai.us/posts/3613024", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, + "#options" : {"ugoira": True}, + "#pattern" : r"https?://.+\.zip$", +}, + +{ + "#url" : "https://danbooru.donmai.us/post/show/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class" : danbooru.DanbooruPostExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/explore/posts/popular", + "#category": ("Danbooru", "danbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, +}, + +{ + "#url" : "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", + "#category": ("Danbooru", "danbooru", "popular"), + "#class" : danbooru.DanbooruPopularExtractor, + "#range" : "1-120", + "#count" : 120, +}, + +) diff --git a/test/results/derpibooru.py b/test/results/derpibooru.py new file mode 100644 index 00000000..7ca6a0de --- /dev/null +++ b/test/results/derpibooru.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#count" : 1, + "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", + + "animated" : False, + "aspect_ratio" : 1.0, + "comment_count" : int, + "created_at" : "2012-01-02T03:12:33Z", + "date" : "dt:2012-01-02 03:12:33", + "deletion_reason" : None, + "description" : "", + "downvotes" : int, + "duplicate_of" : None, + "duration" : 0.04, + "extension" : "png", + "faves" : int, + "first_seen_at" : "2012-01-02T03:12:33Z", + "format" : "png", + "height" : 900, + "hidden_from_users": False, + "id" : 1, + "mime_type" : "image/png", + "name" : "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", + "orig_sha512_hash": None, + "processed" : True, + "representations" : dict, + "score" : int, + "sha512_hash" : "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", + "size" : 860914, + "source_url" : "https://www.deviantart.com/speccysy/art/Afternoon-Flight-215193985", + "spoilered" : False, + "tag_count" : int, + "tag_ids" : list, + "tags" : list, + "thumbnails_generated": True, + "updated_at" : r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", + "uploader" : "Clover the Clever", + "uploader_id" : 211188, + "upvotes" : int, + "view_url" : str, + "width" : 900, + "wilson_score" : float, +}, + +{ + "#url" : "https://derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://www.derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://www.derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://derpibooru.org/search?q=cute", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://derpibooru.org/tags/cute", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", + "#category": ("philomena", "derpibooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://derpibooru.org/galleries/1", + "#category": ("philomena", "derpibooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#pattern" : r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", + + "gallery": { + "description" : "Indexes start at 1 :P", + "id" : 1, + "spoiler_warning": "", + "thumbnail_id" : 1, + "title" : "The Very First Gallery", + "user" : "DeliciousBlackInk", + "user_id" : 365446, + }, +}, + +) diff --git a/test/results/desktopography.py b/test/results/desktopography.py new file mode 100644 index 00000000..c6066692 --- /dev/null +++ b/test/results/desktopography.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import desktopography + + +__tests__ = ( +{ + "#url" : "https://desktopography.net/", + "#category": ("", "desktopography", "site"), + "#class" : desktopography.DesktopographySiteExtractor, +}, + +{ + "#url" : "https://desktopography.net/exhibition-2020/", + "#category": ("", "desktopography", "exhibition"), + "#class" : desktopography.DesktopographyExhibitionExtractor, +}, + +{ + "#url" : "https://desktopography.net/portfolios/new-era/", + "#category": ("", "desktopography", "entry"), + "#class" : desktopography.DesktopographyEntryExtractor, +}, + +) diff --git a/test/results/desuarchive.py b/test/results/desuarchive.py new file mode 100644 index 00000000..8e180652 --- /dev/null +++ b/test/results/desuarchive.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://desuarchive.org/a/thread/159542679/", + "#category": ("foolfuuka", "desuarchive", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", +}, + +{ + "#url" : "https://desuarchive.org/a/", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://desuarchive.org/_/search/text/test/", + "#category": ("foolfuuka", "desuarchive", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://desuarchive.org/a/gallery/5", + "#category": ("foolfuuka", "desuarchive", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/deviantart.py b/test/results/deviantart.py new file mode 100644 index 00000000..87473bb8 --- /dev/null +++ b/test/results/deviantart.py @@ -0,0 +1,732 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import deviantart +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, + "#pattern" : "/shimoda7/gallery$", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$", + "#count" : 5, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/", + "#category": ("", "deviantart", "user"), + "#class" : deviantart.DeviantartUserExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#pattern" : r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", + "#count" : ">= 30", + + "allows_comments" : bool, + "author" : { + "type" : "regular", + "usericon": str, + "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "category_path" : str, + "content" : { + "filesize" : int, + "height" : int, + "src" : str, + "transparency": bool, + "width" : int, + }, + "da_category" : str, + "date" : datetime.datetime, + "deviationid" : str, + "?download_filesize": int, + "extension" : str, + "index" : int, + "is_deleted" : bool, + "is_downloadable" : bool, + "is_favourited" : bool, + "is_mature" : bool, + "preview" : { + "height" : int, + "src" : str, + "transparency": bool, + "width" : int, + }, + "published_time" : int, + "stats" : { + "comments" : int, + "favourites": int, + }, + "target" : dict, + "thumbs" : list, + "title" : str, + "url" : r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", + "username" : "shimoda7", +}, + +{ + "#url" : "https://www.deviantart.com/yakuzafc/gallery", + "#comment" : "group", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#pattern" : r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.deviantart.com/justatest235723/gallery", + "#comment" : "'folders' option (#276)", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#options" : { + "metadata": 1, + "folders" : 1, + "original": 0, + }, + "#count" : 3, + + "description": str, + "folders" : list, + "is_watching": bool, + "license" : str, + "tags" : list, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda8/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/all", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/all/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class" : deviantart.DeviantartGalleryExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", + "#comment" : "user", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 5, +}, + +{ + "#url" : "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", + "#comment" : "group", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : ">= 4", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", + "#comment" : "uuid", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 5, +}, + +{ + "#url" : "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", + "#comment" : "name starts with '_', special characters (#1451)", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#count" : 1, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, +}, + +{ + "#url" : "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, +}, + +{ + "#url" : "https://sta.sh/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.png\?token=.+", + "#count" : 1, + "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", +}, + +{ + "#url" : "https://sta.sh/21jf51j7pzl2", + "#comment" : "multiple stash items", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#options" : {"original": False}, + "#count" : 4, +}, + +{ + "#url" : "https://sta.sh/024t4coz16mi", + "#comment" : "downloadable, but no 'content' field (#307)", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", + "#count" : 1, +}, + +{ + "#url" : "https://sta.sh/215twi387vfj", + "#comment" : "mixed folders and images (#659)", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#options" : {"original": False}, + "#count" : 4, +}, + +{ + "#url" : "https://sta.sh/abcdefghijkl", + "#category": ("", "deviantart", "stash"), + "#class" : deviantart.DeviantartStashExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, + "#options" : { + "metadata": True, + "flat" : False, + }, + "#count" : 1, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/h3813067/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://h3813067.deviantart.com/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class" : deviantart.DeviantartFavoriteExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, + "#options" : {"original": False}, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, + "#options" : {"original": False}, + "#count" : ">= 15", +}, + +{ + "#url" : "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class" : deviantart.DeviantartCollectionExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#sha1_url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#options" : {"journals": "text"}, + "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", +}, + +{ + "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, + "#options" : {"journals": "none"}, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/posts/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/journal/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class" : deviantart.DeviantartJournalExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/t1na/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#count" : 4, + "#sha1_url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074", +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : {"journals": "none"}, + "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/f/[^/]+/[^.]+\.jpg\?token=", + "#count" : 1, +}, + +{ + "#url" : "https://www.deviantart.com/vanillaghosties/posts/statuses", + "#comment" : "shared sta.sh item", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : { + "journals": "none", + "original": False, + }, + "#range" : "5-", + "#count" : 1, + + "index" : int, + "index_base36": r"re:^[0-9a-z]+$", + "url" : r"re:^https://sta.sh", +}, + +{ + "#url" : "https://www.deviantart.com/AndrejSKalin/posts/statuses", + "#comment" : "'deleted' deviations in 'items'", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : { + "journals" : "none", + "original" : 0, + "image-filter": "deviationid[:8] == '147C8B03'", + }, + "#count" : 2, + "#archive" : False, + + "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", +}, + +{ + "#url" : "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class" : deviantart.DeviantartStatusExtractor, + "#options" : {"journals": "text"}, + "#sha1_url": "c8744f7f733a3029116607b826321233c5ca452d", +}, + +{ + "#url" : "https://www.deviantart.com/?order=popular-all-time", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/popular-24-hours/?q=tree+house", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/artisan/popular-all-time/?q=tree", + "#category": ("", "deviantart", "popular"), + "#class" : deviantart.DeviantartPopularExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/tag/nature", + "#category": ("", "deviantart", "tag"), + "#class" : deviantart.DeviantartTagExtractor, + "#options" : {"original": False}, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://www.deviantart.com/watch/deviations", + "#category": ("", "deviantart", "watch"), + "#class" : deviantart.DeviantartWatchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/notifications/watch", + "#category": ("", "deviantart", "watch"), + "#class" : deviantart.DeviantartWatchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/watch/posts", + "#category": ("", "deviantart", "watch-posts"), + "#class" : deviantart.DeviantartWatchPostsExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"comments": True}, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", + + "comments": list, +}, + +{ + "#url" : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", + "#comment" : "wixmp URL rewrite", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/f/[^/]+/[^.]+\.jpg\?token=", +}, + +{ + "#url" : "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", + "#comment" : "GIF (#242)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", +}, + +{ + "#url" : "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", + "#comment" : "Flash animation with GIF preview (#1731)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", + + "filename" : "flash_comic_tutorial_by_yuumei-d3juatd", + "extension": "swf", +}, + +{ + "#url" : "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", + "#comment" : "sta.sh URLs from description (#302)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : { + "extra" : 1, + "original": 0, + }, + "#pattern" : deviantart.DeviantartStashExtractor.pattern, + "#range" : "2-", + "#count" : 4, +}, + +{ + "#url" : "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", + "#comment" : "sta.sh URL from deviation['text_content']['body']['features']", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"extra": 1}, + "#pattern" : r"""text: +|(?:https?://)?sta\.sh/([a-z0-9]+)""", + "#count" : 2, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", + "#comment" : "journal", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : """text: +""", + "#sha1_url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", +}, + +{ + "#url" : "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", + "#comment" : "journal-like post with isJournal == False (#419)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#pattern" : """text: +""", + "#sha1_url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", +}, + +{ + "#url" : "https://deviantart.com/view/904858796/", + "#comment" : "/view/ URLs", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", +}, + +{ + "#url" : "http://www.deviantart.com/view/890672057", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", +}, + +{ + "#url" : "https://www.deviantart.com/view/706871727", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#sha1_content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65", +}, + +{ + "#url" : "https://www.deviantart.com/view/1", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.deviantart.com/deviation/817215762", + "#comment" : "/deviation/ (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://fav.me/ddijrpu", + "#comment" : "fav.me (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://fav.me/dddd", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", + "#comment" : "old-style URLs", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://zzz.deviantart.com/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/view.php?id=14864502", + "#comment" : "old /view/ URLs from the Wayback Machine", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "http://www.deviantart.com/view-full.php?id=100842", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.fxdeviantart.com/view/1234567890", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, +}, + +{ + "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class" : deviantart.DeviantartScrapsExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/search?q=tree", + "#category": ("", "deviantart", "search"), + "#class" : deviantart.DeviantartSearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/search/deviations?order=popular-1-week", + "#category": ("", "deviantart", "search"), + "#class" : deviantart.DeviantartSearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory", + "#category": ("", "deviantart", "gallery-search"), + "#class" : deviantart.DeviantartGallerySearchExtractor, + "#options" : {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", + "#category": ("", "deviantart", "gallery-search"), + "#class" : deviantart.DeviantartGallerySearchExtractor, +}, + +{ + "#url" : "https://www.deviantart.com/shimoda7/about#watching", + "#category": ("", "deviantart", "following"), + "#class" : deviantart.DeviantartFollowingExtractor, + "#pattern" : deviantart.DeviantartUserExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +) diff --git a/test/results/directlink.py b/test/results/directlink.py new file mode 100644 index 00000000..afa43b34 --- /dev/null +++ b/test/results/directlink.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import directlink + + +__tests__ = ( +{ + "#url" : "https://en.wikipedia.org/static/images/project-logos/enwiki.png", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "18c5d00077332e98e53be9fed2ee4be66154b88d", + "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", +}, + +{ + "#url" : "https://example.org/file.webm", + "#comment" : "empty path", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "2d807ed7059d1b532f1bb71dc24b510b80ff943f", + "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", +}, + +{ + "#url" : "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", + "#comment" : "more complex example", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "6fb1061390f8aada3db01cb24b51797c7ee42b31", + "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", +}, + +{ + "#url" : "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", + "#comment" : "percent-encoded characters", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "2627e8140727fdf743f86fe18f69f99a052c9718", + "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", +}, + +{ + "#url" : "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", + "#comment" : "upper case file extension (#296)", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, +}, + +{ + "#url" : "https://räksmörgås.josefsson.org/raksmorgas.jpg", + "#comment" : "internationalized domain name", + "#category": ("", "directlink", ""), + "#class" : directlink.DirectlinkExtractor, + "#sha1_url" : "a65667f670b194afbd1e3ea5e7a78938d36747da", + "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", +}, + +) diff --git a/test/results/drawfriends.py b/test/results/drawfriends.py new file mode 100644 index 00000000..d481fe70 --- /dev/null +++ b/test/results/drawfriends.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "drawfriends", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, +}, + +{ + "#url" : "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "drawfriends", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, +}, + +{ + "#url" : "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", + "#category": ("gelbooru_v01", "drawfriends", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, +}, + +) diff --git a/test/results/dynastyscans.py b/test/results/dynastyscans.py new file mode 100644 index 00000000..bcd85917 --- /dev/null +++ b/test/results/dynastyscans.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import dynastyscans + + +__tests__ = ( +{ + "#url" : "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", + "#category": ("", "dynastyscans", "chapter"), + "#class" : dynastyscans.DynastyscansChapterExtractor, + "#sha1_url" : "dce64e8c504118f1ab4135c00245ea12413896cb", + "#sha1_metadata": "b67599703c27316a2fe4f11c3232130a1904e032", +}, + +{ + "#url" : "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", + "#category": ("", "dynastyscans", "chapter"), + "#class" : dynastyscans.DynastyscansChapterExtractor, + "#sha1_url" : "dbe5bbb74da2edcfb1832895a484e2a40bc8b538", + "#sha1_metadata": "6b674eb3a274999153f6be044973b195008ced2f", +}, + +{ + "#url" : "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", + "#category": ("", "dynastyscans", "manga"), + "#class" : dynastyscans.DynastyscansMangaExtractor, + "#pattern" : dynastyscans.DynastyscansChapterExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://dynasty-scans.com/images?with[]=4930&with[]=5211", + "#category": ("", "dynastyscans", "search"), + "#class" : dynastyscans.DynastyscansSearchExtractor, + "#sha1_url" : "22cf0fb64e12b29e79b0a3d26666086a48f9916a", + "#sha1_metadata": "11cbc555a15528d25567977b8808e10369c4c3ee", +}, + +{ + "#url" : "https://dynasty-scans.com/images", + "#category": ("", "dynastyscans", "search"), + "#class" : dynastyscans.DynastyscansSearchExtractor, + "#range" : "1", + "#count" : 1, +}, + +{ + "#url" : "https://dynasty-scans.com/images/1245", + "#category": ("", "dynastyscans", "image"), + "#class" : dynastyscans.DynastyscansImageExtractor, + "#sha1_url" : "15e54bd94148a07ed037f387d046c27befa043b2", + "#sha1_metadata": "0d8976c2d6fbc9ed6aa712642631b96e456dc37f", +}, + +) diff --git a/test/results/e621.py b/test/results/e621.py new file mode 100644 index 00000000..dd9787cd --- /dev/null +++ b/test/results/e621.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e621.net/posts?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, + "#sha1_url" : "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", +}, + +{ + "#url" : "https://e621.net/post/index/1/anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e621.net/post?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e621.net/pools/73", + "#category": ("E621", "e621", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url" : "1bd09a72715286a79eea3b7f09f51b3493eb579a", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", +}, + +{ + "#url" : "https://e621.net/pool/show/73", + "#category": ("E621", "e621", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e621.net/posts/535", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + + "date": "dt:2007-02-17 19:02:32", +}, + +{ + "#url" : "https://e621.net/posts/3181052", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, + "#options" : {"metadata": "notes,pools"}, + "#pattern" : r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", + + "notes": [{ + "body" : "Little Legends 2", + "created_at" : "2022-05-16T13:58:38.877-04:00", + "creator_id" : 517450, + "creator_name": "EeveeCuddler69", + "height" : 475, + "id" : 321296, + "is_active" : True, + "post_id" : 3181052, + "updated_at" : "2022-05-16T13:59:02.050-04:00", + "version" : 3, + "width" : 809, + "x" : 83, + "y" : 117, +}], + "pools": [{ + "category" : "series", + "created_at" : "2022-02-17T00:29:22.669-05:00", + "creator_id" : 1077440, + "creator_name": "Yeetus90", + "description" : """* "Little Legends":/pools/27971 +* Little Legends 2 +* "Little Legends 3":/pools/27481""", + "id" : 27492, + "is_active" : False, + "name" : "Little Legends 2", + "post_count" : 39, + "post_ids" : list, + "updated_at" : "2022-03-27T06:30:03.382-04:00", +}], +}, + +{ + "#url" : "https://e621.net/post/show/535", + "#category": ("E621", "e621", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e621.net/explore/posts/popular", + "#category": ("E621", "e621", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e621", "popular"), + "#class" : e621.E621PopularExtractor, + "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count" : ">= 70", +}, + +{ + "#url" : "https://e621.net/favorites", + "#category": ("E621", "e621", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +{ + "#url" : "https://e621.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e621", "favorite"), + "#class" : e621.E621FavoriteExtractor, + "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count" : "> 260", +}, + +) diff --git a/test/results/e6ai.py b/test/results/e6ai.py new file mode 100644 index 00000000..291f3b3a --- /dev/null +++ b/test/results/e6ai.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e6ai.net/posts?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/post/index/1/anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/post?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e6ai.net/pools/3", + "#category": ("E621", "e6ai", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", +}, + +{ + "#url" : "https://e6ai.net/pool/show/3", + "#category": ("E621", "e6ai", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e6ai.net/posts/23", + "#category": ("E621", "e6ai", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "3c85a806b3d9eec861948af421fe0e8ad6b8f881", + "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", +}, + +{ + "#url" : "https://e6ai.net/post/show/23", + "#category": ("E621", "e6ai", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e6ai.net/explore/posts/popular", + "#category": ("E621", "e6ai", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e6ai.net/favorites", + "#category": ("E621", "e6ai", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +) diff --git a/test/results/e926.py b/test/results/e926.py new file mode 100644 index 00000000..b046b459 --- /dev/null +++ b/test/results/e926.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import e621 + + +__tests__ = ( +{ + "#url" : "https://e926.net/posts?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, + "#sha1_url" : "12198b275c62ffe2de67cca676c8e64de80c425d", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", +}, + +{ + "#url" : "https://e926.net/post/index/1/anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e926.net/post?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class" : e621.E621TagExtractor, +}, + +{ + "#url" : "https://e926.net/pools/73", + "#category": ("E621", "e926", "pool"), + "#class" : e621.E621PoolExtractor, + "#sha1_url" : "6936f1b6a18c5c25bee7cad700088dbc2503481b", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", +}, + +{ + "#url" : "https://e926.net/pool/show/73", + "#category": ("E621", "e926", "pool"), + "#class" : e621.E621PoolExtractor, +}, + +{ + "#url" : "https://e926.net/posts/535", + "#category": ("E621", "e926", "post"), + "#class" : e621.E621PostExtractor, + "#sha1_url" : "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", +}, + +{ + "#url" : "https://e926.net/post/show/535", + "#category": ("E621", "e926", "post"), + "#class" : e621.E621PostExtractor, +}, + +{ + "#url" : "https://e926.net/explore/posts/popular", + "#category": ("E621", "e926", "popular"), + "#class" : e621.E621PopularExtractor, +}, + +{ + "#url" : "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e926", "popular"), + "#class" : e621.E621PopularExtractor, + "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count" : ">= 70", +}, + +{ + "#url" : "https://e926.net/favorites", + "#category": ("E621", "e926", "favorite"), + "#class" : e621.E621FavoriteExtractor, +}, + +{ + "#url" : "https://e926.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e926", "favorite"), + "#class" : e621.E621FavoriteExtractor, + "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count" : "> 260", +}, + +) diff --git a/test/results/endchan.py b/test/results/endchan.py new file mode 100644 index 00000000..293a8569 --- /dev/null +++ b/test/results/endchan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://endchan.org/yuri/res/193483.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://endchan\.org/\.media/[^.]+(\.\w+)?$", + "#count" : ">= 19", +}, + +{ + "#url" : "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, +}, + +{ + "#url" : "https://endchan.org/yuri/", + "#category": ("lynxchan", "endchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 9", +}, + +{ + "#url" : "https://endchan.org/yuri/catalog.html", + "#category": ("lynxchan", "endchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/erome.py b/test/results/erome.py new file mode 100644 index 00000000..f3269c04 --- /dev/null +++ b/test/results/erome.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import erome + + +__tests__ = ( +{ + "#url" : "https://www.erome.com/a/NQgdlWvk", + "#category": ("", "erome", "album"), + "#class" : erome.EromeAlbumExtractor, + "#pattern" : r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", + "#count" : 1, + + "album_id": "NQgdlWvk", + "num" : 1, + "title" : "porn", + "user" : "yYgWBZw8o8qsMzM", +}, + +{ + "#url" : "https://www.erome.com/a/TdbZ4ogi", + "#category": ("", "erome", "album"), + "#class" : erome.EromeAlbumExtractor, + "#pattern" : r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", + "#count" : 6, + + "album_id": "TdbZ4ogi", + "num" : int, + "title" : "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", + "user" : "yYgWBZw8o8qsMzM", +}, + +{ + "#url" : "https://www.erome.com/yYgWBZw8o8qsMzM", + "#category": ("", "erome", "user"), + "#class" : erome.EromeUserExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://www.erome.com/search?q=cute", + "#category": ("", "erome", "search"), + "#class" : erome.EromeSearchExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +) diff --git a/test/results/exhentai.py b/test/results/exhentai.py new file mode 100644 index 00000000..293449c8 --- /dev/null +++ b/test/results/exhentai.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import exhentai +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://exhentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#sha1_content": [ + "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", + "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", + ], + + "cost" : int, + "date" : "dt:2018-03-18 20:14:00", + "eh_category" : "Non-H", + "expunged" : False, + "favorites" : r"re:^[12]\d$", + "filecount" : "4", + "filesize" : 1488978, + "gid" : 1200119, + "height" : int, + "image_token" : r"re:[0-9a-f]{10}", + "lang" : "ja", + "language" : "Japanese", + "parent" : "", + "rating" : r"re:\d\.\d+", + "size" : int, + "tags" : [ + "parody:komi-san wa komyushou desu.", + "character:shouko komi", + "group:seventh lowlife", + "other:sample", + ], + "thumb" : "https://exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", + "title" : "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", + "title_jpn" : "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", + "token" : "d55c44d3d0", + "torrentcount": "0", + "uploader" : "klorpa", + "width" : int, +}, + +{ + "#url" : "https://exhentai.org/g/960461/4f0e369d82/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "http://exhentai.org/g/962698/7f02358e00/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://exhentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#count" : 2, +}, + +{ + "#url" : "https://e-hentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, + "#options" : {"original": False}, + "#count" : 2, +}, + +{ + "#url" : "https://g.e-hentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class" : exhentai.ExhentaiGalleryExtractor, +}, + +{ + "#url" : "https://e-hentai.org/?f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/?f_cats=767&f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/tag/parody:touhou+project", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, +}, + +{ + "#url" : "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", + "#category": ("", "exhentai", "search"), + "#class" : exhentai.ExhentaiSearchExtractor, + "#pattern" : exhentai.ExhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, + + "gallery_id" : int, + "gallery_token": r"re:^[0-9a-f]{10}$", +}, + +{ + "#url" : "https://e-hentai.org/favorites.php", + "#category": ("", "exhentai", "favorite"), + "#class" : exhentai.ExhentaiFavoriteExtractor, + "#pattern" : r"https?://e-hentai\.org/g/1200119/d55c44d3d0", + "#count" : 1, +}, + +{ + "#url" : "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", + "#category": ("", "exhentai", "favorite"), + "#class" : exhentai.ExhentaiFavoriteExtractor, +}, + +) diff --git a/test/results/fallenangels.py b/test/results/fallenangels.py new file mode 100644 index 00000000..e2167bb4 --- /dev/null +++ b/test/results/fallenangels.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fallenangels + + +__tests__ = ( +{ + "#url" : "https://manga.fascans.com/manga/chronos-ruler/20/1", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "4604a7914566cc2da0ff789aa178e2d1c8c241e3", + "#sha1_metadata": "2dfcc50020e32cd207be88e2a8fac0933e36bdfb", +}, + +{ + "#url" : "http://truyen.fascans.com/manga/hungry-marie/8", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "1f923d9cb337d5e7bbf4323719881794a951c6ae", + "#sha1_metadata": "2bdb7334c0e3eceb9946ffd3132df679b4a94f6a", +}, + +{ + "#url" : "http://manga.fascans.com/manga/rakudai-kishi-no-eiyuutan/19.5", + "#category": ("", "fallenangels", "chapter"), + "#class" : fallenangels.FallenangelsChapterExtractor, + "#sha1_url" : "273f6863966c83ea79ad5846a2866e08067d3f0e", + "#sha1_metadata": "d1065685bfe0054c4ff2a0f20acb089de4cec253", +}, + +{ + "#url" : "https://manga.fascans.com/manga/chronos-ruler", + "#category": ("", "fallenangels", "manga"), + "#class" : fallenangels.FallenangelsMangaExtractor, + "#sha1_url" : "eea07dd50f5bc4903aa09e2cc3e45c7241c9a9c2", + "#sha1_metadata": "c414249525d4c74ad83498b3c59a813557e59d7e", +}, + +{ + "#url" : "https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", + "#category": ("", "fallenangels", "manga"), + "#class" : fallenangels.FallenangelsMangaExtractor, + "#sha1_url" : "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b", + "#sha1_metadata": "2d2a2a5d9ea5925eb9a47bb13d848967f3af086c", +}, + +) diff --git a/test/results/fanbox.py b/test/results/fanbox.py new file mode 100644 index 00000000..78f7fe54 --- /dev/null +++ b/test/results/fanbox.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fanbox + + +__tests__ = ( +{ + "#url" : "https://xub.fanbox.cc", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, + "#range" : "1-15", + "#count" : ">= 15", + + "creatorId": "xub", + "tags" : list, + "title" : str, +}, + +{ + "#url" : "https://xub.fanbox.cc/posts", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/posts", + "#category": ("", "fanbox", "creator"), + "#class" : fanbox.FanboxCreatorExtractor, +}, + +{ + "#url" : "https://www.fanbox.cc/@xub/posts/1910054", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#count" : 3, + + "title" : "えま★おうがすと", + "tags" : list, + "hasAdultContent": True, + "isCoverImage" : False, +}, + +{ + "#url" : "https://nekoworks.fanbox.cc/posts/915", + "#comment" : "entry post type, image embedded in html of the post", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#count" : 2, + + "title" : "【SAYORI FAN CLUB】お届け内容", + "tags" : list, + "html" : str, + "hasAdultContent": True, +}, + +{ + "#url" : "https://steelwire.fanbox.cc/posts/285502", + "#comment" : "article post type, imageMap, 2 twitter embeds, fanbox embed", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#options" : {"embeds": True}, + "#count" : 10, + + "title" : "イラスト+SS｜義足の炭鉱少年が義足を見せてくれるだけ 【全体公開版】", + "tags" : list, + "articleBody" : dict, + "hasAdultContent": True, +}, + +{ + "#url" : "https://www.fanbox.cc/@official-en/posts/4326303", + "#comment" : "'content' metadata (#3020)", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + + "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", +}, + +{ + "#url" : "https://mochirong.fanbox.cc/posts/3746116", + "#comment" : "imageMap file order (#2718)", + "#category": ("", "fanbox", "post"), + "#class" : fanbox.FanboxPostExtractor, + "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", +}, + +{ + "#url" : "https://www.pixiv.net/fanbox/creator/52336352", + "#category": ("", "fanbox", "redirect"), + "#class" : fanbox.FanboxRedirectExtractor, + "#pattern" : fanbox.FanboxCreatorExtractor.pattern, +}, + +) diff --git a/test/results/fanleaks.py b/test/results/fanleaks.py new file mode 100644 index 00000000..c420e0ca --- /dev/null +++ b/test/results/fanleaks.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fanleaks +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://fanleaks.club/selti/880", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#pattern" : r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", + + "model_id": "selti", + "model" : "Selti", + "id" : 880, + "type" : "photo", +}, + +{ + "#url" : "https://fanleaks.club/daisy-keech/1038", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#pattern" : r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", + + "model_id": "daisy-keech", + "model" : "Daisy Keech", + "id" : 1038, + "type" : "video", +}, + +{ + "#url" : "https://fanleaks.club/hannahowo/000", + "#category": ("", "fanleaks", "post"), + "#class" : fanleaks.FanleaksPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://fanleaks.club/hannahowo", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, + "#pattern" : r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://fanleaks.club/belle-delphine", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, + "#pattern" : r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://fanleaks.club/daisy-keech", + "#category": ("", "fanleaks", "model"), + "#class" : fanleaks.FanleaksModelExtractor, +}, + +) diff --git a/test/results/fantia.py b/test/results/fantia.py new file mode 100644 index 00000000..706b89d6 --- /dev/null +++ b/test/results/fantia.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fantia + + +__tests__ = ( +{ + "#url" : "https://fantia.jp/fanclubs/6939", + "#category": ("", "fantia", "creator"), + "#class" : fantia.FantiaCreatorExtractor, + "#range" : "1-25", + "#count" : ">= 25", + + "fanclub_user_id": 52152, + "tags" : list, + "title" : str, +}, + +{ + "#url" : "https://fantia.jp/posts/1166373", + "#category": ("", "fantia", "post"), + "#class" : fantia.FantiaPostExtractor, + "#pattern" : r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", + + "blogpost_text" : r"re:^$|This is a test.\n\nThis is a test.\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", + "comment" : """ + +""", + "content_category": r"re:thumb|blog|photo_gallery", + "content_comment" : str, + "content_filename": r"re:|", + "content_title" : r"re:Test (Blog Content \d+|Image Gallery)|thumb", + "date" : "dt:2022-03-09 16:46:12", + "fanclub_id" : 356320, + "fanclub_name" : "Test Fantia", + "fanclub_url" : "https://fantia.jp/fanclubs/356320", + "fanclub_user_id" : 7487131, + "fanclub_user_name": "2022/03/08 15:13:52の名無し", + "file_url" : str, + "filename" : str, + "num" : int, + "plan" : dict, + "post_id" : 1166373, + "post_title" : "Test Fantia Post", + "post_url" : "https://fantia.jp/posts/1166373", + "posted_at" : "Thu, 10 Mar 2022 01:46:12 +0900", + "rating" : "general", + "tags" : [], +}, + +{ + "#url" : "https://fantia.jp/posts/508363", + "#category": ("", "fantia", "post"), + "#class" : fantia.FantiaPostExtractor, + "#count" : 6, + + "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", + "tags" : list, + "rating" : "adult", + "post_id" : 508363, +}, + +) diff --git a/test/results/fapachi.py b/test/results/fapachi.py new file mode 100644 index 00000000..8907e7f8 --- /dev/null +++ b/test/results/fapachi.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fapachi + + +__tests__ = ( +{ + "#url" : "https://fapachi.com/sonson/media/0082", + "#comment" : "NSFW", + "#category": ("", "fapachi", "post"), + "#class" : fapachi.FapachiPostExtractor, + "#pattern" : r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", + + "user": "sonson", + "id" : "0082", +}, + +{ + "#url" : "https://fapachi.com/ferxiita/media/0159", + "#comment" : "NSFW", + "#category": ("", "fapachi", "post"), + "#class" : fapachi.FapachiPostExtractor, +}, + +{ + "#url" : "https://fapachi.com/sonson", + "#category": ("", "fapachi", "user"), + "#class" : fapachi.FapachiUserExtractor, + "#pattern" : fapachi.FapachiPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://fapachi.com/ferxiita/page/3", + "#category": ("", "fapachi", "user"), + "#class" : fapachi.FapachiUserExtractor, +}, + +) diff --git a/test/results/fapello.py b/test/results/fapello.py new file mode 100644 index 00000000..2b8381ca --- /dev/null +++ b/test/results/fapello.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fapello +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://fapello.com/carrykey/530/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#pattern" : r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", + + "model" : "carrykey", + "id" : 530, + "type" : "photo", + "thumbnail": "", +}, + +{ + "#url" : "https://fapello.com/vladislava-661/693/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#pattern" : r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", + + "model" : "vladislava-661", + "id" : 693, + "type" : "video", + "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", +}, + +{ + "#url" : "https://fapello.com/carrykey/000/", + "#category": ("", "fapello", "post"), + "#class" : fapello.FapelloPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://fapello.com/hyoon/", + "#category": ("", "fapello", "model"), + "#class" : fapello.FapelloModelExtractor, + "#pattern" : fapello.FapelloPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://fapello.com/kobaebeefboo/", + "#category": ("", "fapello", "model"), + "#class" : fapello.FapelloModelExtractor, +}, + +{ + "#url" : "https://fapello.com/top-likes/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, + "#pattern" : fapello.FapelloModelExtractor.pattern, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://fapello.com/videos/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, + "#pattern" : fapello.FapelloPostExtractor.pattern, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://fapello.com/top-followers/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/trending/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/popular_videos/twelve_hours/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +{ + "#url" : "https://fapello.com/popular_videos/week/", + "#category": ("", "fapello", "path"), + "#class" : fapello.FapelloPathExtractor, +}, + +) diff --git a/test/results/fappic.py b/test/results/fappic.py new file mode 100644 index 00000000..b89c4e29 --- /dev/null +++ b/test/results/fappic.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class" : imagehosts.FappicImageExtractor, +}, + +{ + "#url" : "https://www.fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class" : imagehosts.FappicImageExtractor, + "#pattern" : r"https://img\d+\.fappic\.com/img/\w+/test\.png", + "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/fashionnova.py b/test/results/fashionnova.py new file mode 100644 index 00000000..4225264f --- /dev/null +++ b/test/results/fashionnova.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses/?page=1", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/mini-dresses#1", + "#category": ("shopify", "fashionnova", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.fashionnova.com/products/essential-slide-red", + "#category": ("shopify", "fashionnova", "product"), + "#class" : shopify.ShopifyProductExtractor, + "#pattern" : r"https?://cdn\d*\.shopify.com/", + "#count" : 3, +}, + +{ + "#url" : "https://www.fashionnova.com/collections/flats/products/name", + "#category": ("shopify", "fashionnova", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/fireden.py b/test/results/fireden.py new file mode 100644 index 00000000..48549e18 --- /dev/null +++ b/test/results/fireden.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://boards.fireden.net/sci/thread/11264294/", + "#category": ("foolfuuka", "fireden", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", +}, + +{ + "#url" : "https://boards.fireden.net/sci/", + "#category": ("foolfuuka", "fireden", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://boards.fireden.net/_/search/text/test/", + "#category": ("foolfuuka", "fireden", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://boards.fireden.net/sci/gallery/6", + "#category": ("foolfuuka", "fireden", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/flickr.py b/test/results/flickr.py new file mode 100644 index 00000000..5f1fee07 --- /dev/null +++ b/test/results/flickr.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import flickr +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#sha1_content": [ + "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", + "0821a28ee46386e85b02b67cf2720063440a228c", + ], + + "comments" : int, + "description": str, + "extension" : "jpg", + "filename" : "16089302239_de18cd8017_b", + "id" : 16089302239, + "height" : 683, + "label" : "Large", + "media" : "photo", + "url" : str, + "views" : int, + "width" : 1024, +}, + +{ + "#url" : "https://secure.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://m.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, +}, + +{ + "#url" : "https://www.flickr.com/photos/145617051@N08/46733161535", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#count" : 1, + + "media": "video", +}, + +{ + "#url" : "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://flic.kr/p/FPVo9U", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, +}, + +{ + "#url" : "https://www.flickr.com/photos/zzz/16089302238", + "#category": ("", "flickr", "image"), + "#class" : flickr.FlickrImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/albums/72157633471741607", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 6, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, + "#pattern" : flickr.FlickrAlbumExtractor.pattern, + "#count" : 2, +}, + +{ + "#url" : "https://secure.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, +}, + +{ + "#url" : "https://m.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class" : flickr.FlickrAlbumExtractor, +}, + +{ + "#url" : "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", + "#category": ("", "flickr", "gallery"), + "#class" : flickr.FlickrGalleryExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.flickr.com/groups/bird_headshots/", + "#category": ("", "flickr", "group"), + "#class" : flickr.FlickrGroupExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : "> 150", +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/", + "#category": ("", "flickr", "user"), + "#class" : flickr.FlickrUserExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 28, +}, + +{ + "#url" : "https://www.flickr.com/photos/shona_s/favorites", + "#category": ("", "flickr", "favorite"), + "#class" : flickr.FlickrFavoriteExtractor, + "#pattern" : flickr.FlickrImageExtractor.pattern, + "#count" : 4, +}, + +{ + "#url" : "https://flickr.com/search/?text=mountain", + "#category": ("", "flickr", "search"), + "#class" : flickr.FlickrSearchExtractor, +}, + +{ + "#url" : "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", + "#category": ("", "flickr", "search"), + "#class" : flickr.FlickrSearchExtractor, +}, + +) diff --git a/test/results/foalcon.py b/test/results/foalcon.py new file mode 100644 index 00000000..7639c0df --- /dev/null +++ b/test/results/foalcon.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import szurubooru + + +__tests__ = ( +{ + "#url" : "https://booru.foalcon.com/posts/query=simple_background", + "#category": ("szurubooru", "foalcon", "tag"), + "#class" : szurubooru.SzurubooruTagExtractor, + "#pattern" : r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#range" : "1-150", + "#count" : 150, +}, + +{ + "#url" : "https://booru.foalcon.com/post/30092", + "#category": ("szurubooru", "foalcon", "post"), + "#class" : szurubooru.SzurubooruPostExtractor, + "#pattern" : r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", + "#sha1_url" : "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", + "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", +}, + +) diff --git a/test/results/furaffinity.py b/test/results/furaffinity.py new file mode 100644 index 00000000..187029bb --- /dev/null +++ b/test/results/furaffinity.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import furaffinity + + +__tests__ = ( +{ + "#url" : "https://www.furaffinity.net/gallery/mirlinthloth/", + "#category": ("", "furaffinity", "gallery"), + "#class" : furaffinity.FuraffinityGalleryExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, +}, + +{ + "#url" : "https://www.furaffinity.net/scraps/mirlinthloth/", + "#category": ("", "furaffinity", "scraps"), + "#class" : furaffinity.FuraffinityScrapsExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", + "#count" : ">= 3", +}, + +{ + "#url" : "https://www.furaffinity.net/favorites/mirlinthloth/", + "#category": ("", "furaffinity", "favorite"), + "#class" : furaffinity.FuraffinityFavoriteExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, + + "favorite_id": int, +}, + +{ + "#url" : "https://www.furaffinity.net/search/?q=cute", + "#category": ("", "furaffinity", "search"), + "#class" : furaffinity.FuraffinitySearchExtractor, + "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range" : "45-50", + "#count" : 6, +}, + +{ + "#url" : "https://www.furaffinity.net/search/?q=leaf&range=1day", + "#comment" : "first page of search results (#2402)", + "#category": ("", "furaffinity", "search"), + "#class" : furaffinity.FuraffinitySearchExtractor, + "#range" : "1-3", + "#count" : 3, +}, + +{ + "#url" : "https://www.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", + + "artist" : "mirlinthloth", + "artist_url" : "mirlinthloth", + "date" : "dt:2016-11-27 17:24:06", + "description": "A Song made playing the game Cosmic DJ.", + "extension" : "mp3", + "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", + "id" : 21835115, + "tags" : list, + "title" : "Bude's 4 Ever", + "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", + "user" : "mirlinthloth", + "views" : int, + "favorites" : int, + "comments" : int, + "rating" : "General", + "fa_category": "Music", + "theme" : "All", + "species" : "Unspecified / Any", + "gender" : "Any", + "width" : 120, + "height" : 120, +}, + +{ + "#url" : "https://www.furaffinity.net/view/42166511/", + "#comment" : "'external' option (#1492)", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + "#options" : {"external": True}, + "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", + "#count" : 2, +}, + +{ + "#url" : "https://www.furaffinity.net/view/45331225/", + "#comment" : "no tags (#2277)", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, + + "artist" : "Kota_Remminders", + "artist_url" : "kotaremminders", + "date" : "dt:2022-01-03 17:49:33", + "fa_category": "Adoptables", + "filename" : "1641232173.kotaremminders_chidopts1", + "gender" : "Any", + "height" : 905, + "id" : 45331225, + "rating" : "General", + "species" : "Unspecified / Any", + "tags" : [], + "theme" : "All", + "title" : "REMINDER", + "width" : 1280, +}, + +{ + "#url" : "https://furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://sfw.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://www.furaffinity.net/full/21835115/", + "#category": ("", "furaffinity", "post"), + "#class" : furaffinity.FuraffinityPostExtractor, +}, + +{ + "#url" : "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class" : furaffinity.FuraffinityUserExtractor, + "#pattern" : "/gallery/mirlinthloth/$", +}, + +{ + "#url" : "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class" : furaffinity.FuraffinityUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "/(gallery|scraps|favorites)/mirlinthloth/$", + "#count" : 3, +}, + +{ + "#url" : "https://www.furaffinity.net/watchlist/by/mirlinthloth/", + "#category": ("", "furaffinity", "following"), + "#class" : furaffinity.FuraffinityFollowingExtractor, + "#pattern" : furaffinity.FuraffinityUserExtractor.pattern, + "#range" : "176-225", + "#count" : 50, +}, + +) diff --git a/test/results/furbooru.py b/test/results/furbooru.py new file mode 100644 index 00000000..682aa005 --- /dev/null +++ b/test/results/furbooru.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://furbooru.org/images/1", + "#category": ("philomena", "furbooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", +}, + +{ + "#url" : "https://furbooru.org/search?q=cute", + "#category": ("philomena", "furbooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://furbooru.org/galleries/27", + "#category": ("philomena", "furbooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#count" : ">= 13", +}, + +) diff --git a/test/results/fuskator.py b/test/results/fuskator.py new file mode 100644 index 00000000..cd68c9c2 --- /dev/null +++ b/test/results/fuskator.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import fuskator + + +__tests__ = ( +{ + "#url" : "https://fuskator.com/thumbs/d0GnIzXrSKU/", + "#category": ("", "fuskator", "gallery"), + "#class" : fuskator.FuskatorGalleryExtractor, + "#pattern" : r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", + "#count" : 22, + + "gallery_id" : 473023, + "gallery_hash": "d0GnIzXrSKU", + "title" : r"re:Shaved Brunette Babe Maria Ryabushkina with ", + "views" : int, + "score" : float, + "count" : 22, + "tags" : list, +}, + +{ + "#url" : "https://fuskator.com/expanded/gXpKzjgIidA/index.html", + "#category": ("", "fuskator", "gallery"), + "#class" : fuskator.FuskatorGalleryExtractor, +}, + +{ + "#url" : "https://fuskator.com/search/red_swimsuit/", + "#category": ("", "fuskator", "search"), + "#class" : fuskator.FuskatorSearchExtractor, + "#pattern" : fuskator.FuskatorGalleryExtractor.pattern, + "#count" : ">= 40", +}, + +{ + "#url" : "https://fuskator.com/page/3/swimsuit/quality/", + "#category": ("", "fuskator", "search"), + "#class" : fuskator.FuskatorSearchExtractor, +}, + +) diff --git a/test/results/gelbooru.py b/test/results/gelbooru.py new file mode 100644 index 00000000..8d9ead27 --- /dev/null +++ b/test/results/gelbooru.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru + + +__tests__ = ( +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", + "#category": ("booru", "gelbooru", "tag"), + "#class" : gelbooru.GelbooruTagExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", + "#category": ("booru", "gelbooru", "tag"), + "#class" : gelbooru.GelbooruTagExtractor, + "#pattern" : r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", + "#range" : "196-204", + "#count" : 9, + "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=pool&s=show&id=761", + "#category": ("booru", "gelbooru", "pool"), + "#class" : gelbooru.GelbooruPoolExtractor, + "#count" : 6, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=279415", + "#category": ("booru", "gelbooru", "favorite"), + "#class" : gelbooru.GelbooruFavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#count" : 1, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?s=view&page=post&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&id=313638&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?s=view&id=313638&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?id=313638&page=post&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?id=313638&s=view&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=6018318", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", + + "tags_artist" : "kirisaki_shuusei", + "tags_character": str, + "tags_copyright": "vocaloid", + "tags_general" : str, + "tags_metadata" : str, +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5938076", + "#comment" : "video", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#pattern" : r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", + "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", +}, + +{ + "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5997331", + "#comment" : "notes", + "#category": ("booru", "gelbooru", "post"), + "#class" : gelbooru.GelbooruPostExtractor, + "#options" : {"notes": True}, + + "notes": [ + { + "body" : "Look over this way when you talk~", + "height": 553, + "width" : 246, + "x" : 35, + "y" : 72, + }, + { + "body" : """Hey~ +Are you listening~?""", + "height": 557, + "width" : 246, + "x" : 1233, + "y" : 109, + }, + ], +}, + +{ + "#url" : "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", + "#category": ("booru", "gelbooru", "redirect"), + "#class" : gelbooru.GelbooruRedirectExtractor, + "#pattern" : r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", +}, + +) diff --git a/test/results/generic.py b/test/results/generic.py new file mode 100644 index 00000000..5ec0400e --- /dev/null +++ b/test/results/generic.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import generic + + +__tests__ = ( +{ + "#url" : "generic:https://www.nongnu.org/lzip/", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, + "#count" : 1, + "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", + + "description": "Lossless data compressor", + "imageurl" : "https://www.nongnu.org/lzip/lzip.png", + "keywords" : "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", + "pageurl" : "https://www.nongnu.org/lzip/", +}, + +{ + "#url" : "generic:https://räksmörgås.josefsson.org/", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, + "#pattern" : "^https://räksmörgås.josefsson.org/", + "#count" : 2, +}, + +{ + "#url" : "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +{ + "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", ""), + "#class" : generic.GenericExtractor, +}, + +) diff --git a/test/results/giantessbooru.py b/test/results/giantessbooru.py new file mode 100644 index 00000000..5909c191 --- /dev/null +++ b/test/results/giantessbooru.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://giantessbooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://giantessbooru\.com/index\.php\?q=/image/\d+\.jpg", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://giantessbooru.com/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, +}, + +{ + "#url" : "https://giantessbooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://giantessbooru\.com/index\.php\?q=/image/41\.jpg", + "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", + + "extension": "jpg", + "file_url" : "https://giantessbooru.com/index.php?q=/image/41.jpg", + "filename" : "41", + "height" : 0, + "id" : 41, + "md5" : "", + "size" : 0, + "tags" : "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", + "width" : 1387, +}, + +{ + "#url" : "https://giantessbooru.com/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, +}, + +) diff --git a/test/results/gofile.py b/test/results/gofile.py new file mode 100644 index 00000000..b7d26702 --- /dev/null +++ b/test/results/gofile.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gofile + + +__tests__ = ( +{ + "#url" : "https://gofile.io/d/k6BomI", + "#category": ("", "gofile", "folder"), + "#class" : gofile.GofileFolderExtractor, + "#pattern" : r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", + + "createTime" : int, + "directLink" : r"re:https://store5.gofile.io/download/direct/.+", + "downloadCount": int, + "extension" : "png", + "filename" : "test-テスト-%22&!", + "folder" : { + "childs" : [ + "b0367d79-b8ba-407f-8342-aaf8eb815443", + "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + ], + "code" : "k6BomI", + "createTime" : 1654076165, + "id" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "name" : "root", + "public" : True, + "totalDownloadCount": int, + "totalSize" : 182, + "type" : "folder", + }, + "id" : r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", + "link" : r"re:https://store5.gofile.io/download/.+\.png", + "md5" : r"re:[0-9a-f]{32}", + "mimetype" : "image/png", + "name" : "test-テスト-%22&!.png", + "num" : int, + "parentFolder" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "serverChoosen": "store5", + "size" : 182, + "thumbnail" : r"re:https://store5.gofile.io/download/.+\.png", + "type" : "file", +}, + +{ + "#url" : "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + "#category": ("", "gofile", "folder"), + "#class" : gofile.GofileFolderExtractor, + "#options" : {"website-token": None}, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/hbrowse.py b/test/results/hbrowse.py new file mode 100644 index 00000000..c4b3dcd6 --- /dev/null +++ b/test/results/hbrowse.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hbrowse + + +__tests__ = ( +{ + "#url" : "https://www.hbrowse.com/10363/c00000", + "#category": ("", "hbrowse", "chapter"), + "#class" : hbrowse.HbrowseChapterExtractor, + "#sha1_url" : "6feefbc9f4b98e20d8425ddffa9dd111791dc3e6", + "#sha1_metadata": "274996f6c809e5250b6ff3abbc5147e29f89d9a5", + "#sha1_content" : "44578ebbe176c2c27434966aef22945787e2781e", +}, + +{ + "#url" : "https://www.hbrowse.com/10363", + "#category": ("", "hbrowse", "manga"), + "#class" : hbrowse.HbrowseMangaExtractor, + "#sha1_url" : "b89682bfb86c11d2af0dc47463804ec3ac4aadd6", + "#sha1_metadata": "4b15fda1858a69de1fbf5afddfe47dd893397312", +}, + +) diff --git a/test/results/hentai2read.py b/test/results/hentai2read.py new file mode 100644 index 00000000..01349c2e --- /dev/null +++ b/test/results/hentai2read.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentai2read + + +__tests__ = ( +{ + "#url" : "https://hentai2read.com/amazon_elixir/1/", + "#category": ("", "hentai2read", "chapter"), + "#class" : hentai2read.Hentai2readChapterExtractor, + "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", +}, + +{ + "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", + "#category": ("", "hentai2read", "chapter"), + "#class" : hentai2read.Hentai2readChapterExtractor, + "#pattern" : r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", + "#count" : 36, + + "author" : "Kurisu", + "chapter" : 2, + "chapter_id" : 75152, + "chapter_minor": ".5", + "count" : 36, + "lang" : "en", + "language" : "English", + "manga" : "Popuni Kei Joshi Panic!", + "manga_id" : 13088, + "page" : int, + "title" : "Popuni Kei Joshi Panic! 2.5", + "type" : "Original", +}, + +{ + "#url" : "https://hentai2read.com/amazon_elixir/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#sha1_url" : "273073752d418ec887d7f7211e42b832e8c403ba", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", +}, + +{ + "#url" : "https://hentai2read.com/oshikage_riot/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#sha1_url" : "6595f920a3088a15c2819c502862d45f8eb6bea6", + "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", +}, + +{ + "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/", + "#category": ("", "hentai2read", "manga"), + "#class" : hentai2read.Hentai2readMangaExtractor, + "#pattern" : hentai2read.Hentai2readChapterExtractor.pattern, + "#range" : "2-3", + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": ".5", + "lang" : "en", + "language" : "English", + "manga" : "Popuni Kei Joshi Panic!", + "manga_id" : 13088, + "title" : str, + "type" : "Original", +}, + +) diff --git a/test/results/hentaicosplays.py b/test/results/hentaicosplays.py new file mode 100644 index 00000000..9760aee8 --- /dev/null +++ b/test/results/hentaicosplays.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaicosplays + + +__tests__ = ( +{ + "#url" : "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.hentai-cosplays.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 18, + "site" : "hentai-cosplays", + "slug" : "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", +}, + +{ + "#url" : "https://fr.porn-images-xxx.com/image/enako-enako-24/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 11, + "site" : "porn-images-xxx", + "title": str, +}, + +{ + "#url" : "https://ja.hentai-img.com/image/hollow-cora-502/", + "#category": ("", "hentaicosplays", "gallery"), + "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern" : r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", + + "count": 2, + "site" : "hentai-img", + "title": str, +}, + +) diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py new file mode 100644 index 00000000..4dc4515d --- /dev/null +++ b/test/results/hentaifoundry.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaifoundry +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/profile", + "#category": ("", "hentaifoundry", "user"), + "#class" : hentaifoundry.HentaifoundryUserExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura", + "#category": ("", "hentaifoundry", "pictures"), + "#class" : hentaifoundry.HentaifoundryPicturesExtractor, + "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", + "#category": ("", "hentaifoundry", "pictures"), + "#class" : hentaifoundry.HentaifoundryPicturesExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", + "#category": ("", "hentaifoundry", "scraps"), + "#class" : hentaifoundry.HentaifoundryScrapsExtractor, + "#sha1_url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", + "#category": ("", "hentaifoundry", "scraps"), + "#class" : hentaifoundry.HentaifoundryScrapsExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", + "#category": ("", "hentaifoundry", "favorite"), + "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, + "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", +}, + +{ + "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", + "#category": ("", "hentaifoundry", "favorite"), + "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/recent/2018-09-20", + "#category": ("", "hentaifoundry", "recent"), + "#class" : hentaifoundry.HentaifoundryRecentExtractor, + "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range" : "20-30", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/popular", + "#category": ("", "hentaifoundry", "popular"), + "#class" : hentaifoundry.HentaifoundryPopularExtractor, + "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range" : "20-30", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, + "#sha1_url" : "fbf2fd74906738094e2575d2728e8dc3de18a8a3", + "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", + + "artist" : "Tenpura", + "date" : "dt:2016-02-22 14:41:19", + "description": "Thank you!", + "height" : 700, + "index" : 407501, + "media" : "Other digital art", + "ratings" : [ + "Sexual content", + "Contains female nudity", + ], + "score" : int, + "tags" : [ + "collection", + "kancolle", + "kantai", + "shimakaze", + ], + "title" : "shimakaze", + "user" : "Tenpura", + "views" : int, + "width" : 495, +}, + +{ + "#url" : "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, + "#pattern" : "http://pictures.hentai-foundry.com/t/Tenpura/407501/", +}, + +{ + "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, +}, + +{ + "#url" : "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", + "#category": ("", "hentaifoundry", "image"), + "#class" : hentaifoundry.HentaifoundryImageExtractor, +}, + +{ + "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35", + "#category": ("", "hentaifoundry", "stories"), + "#class" : hentaifoundry.HentaifoundryStoriesExtractor, + "#count" : ">= 35", + + "author" : "SnowWolf35", + "chapters" : int, + "comments" : int, + "date" : datetime.datetime, + "description": str, + "index" : int, + "rating" : int, + "ratings" : list, + "status" : r"re:(Inc|C)omplete", + "title" : str, + "user" : "SnowWolf35", + "views" : int, + "words" : int, +}, + +{ + "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", + "#category": ("", "hentaifoundry", "story"), + "#class" : hentaifoundry.HentaifoundryStoryExtractor, + "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", + + "title": "Overwatch High Chapter Voting Location", +}, + +) diff --git a/test/results/hentaifox.py b/test/results/hentaifox.py new file mode 100644 index 00000000..123bd277 --- /dev/null +++ b/test/results/hentaifox.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaifox + + +__tests__ = ( +{ + "#url" : "https://hentaifox.com/gallery/56622/", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + "#pattern" : r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", + "#count" : 24, + "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", +}, + +{ + "#url" : "https://hentaifox.com/gallery/630/", + "#comment" : "'split_tag' element (#1378)", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + + "artist" : [ + "beti", + "betty", + "magi", + "mimikaki", + ], + "characters": [ + "aerith gainsborough", + "tifa lockhart", + "yuffie kisaragi", + ], + "count" : 32, + "gallery_id": 630, + "group" : ["cu-little2"], + "parody" : [ + "darkstalkers | vampire", + "final fantasy vii", + ], + "tags" : [ + "femdom", + "fingering", + "masturbation", + "yuri", + ], + "title" : "Cu-Little Bakanya～", + "type" : "doujinshi", +}, + +{ + "#url" : "https://hentaifox.com/gallery/35261/", + "#comment" : "email-protected title (#4201)", + "#category": ("", "hentaifox", "gallery"), + "#class" : hentaifox.HentaifoxGalleryExtractor, + + "gallery_id": 35261, + "title" : "ManageM@ster!", + "artist" : ["haritama hiroki"], + "group" : ["studio n.ball"], +}, + +{ + "#url" : "https://hentaifox.com/parody/touhou-project/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/character/reimu-hakurei/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/artist/distance/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/search/touhou/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/group/v-slash/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, +}, + +{ + "#url" : "https://hentaifox.com/tag/heterochromia/", + "#category": ("", "hentaifox", "search"), + "#class" : hentaifox.HentaifoxSearchExtractor, + "#pattern" : hentaifox.HentaifoxGalleryExtractor.pattern, + "#count" : ">= 60", + + "url" : str, + "gallery_id": int, + "title" : str, +}, + +) diff --git a/test/results/hentaihand.py b/test/results/hentaihand.py new file mode 100644 index 00000000..3d9a34b6 --- /dev/null +++ b/test/results/hentaihand.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaihand + + +__tests__ = ( +{ + "#url" : "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", + "#category": ("", "hentaihand", "gallery"), + "#class" : hentaihand.HentaihandGalleryExtractor, + "#pattern" : r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", + "#count" : 50, + + "artists" : ["Takumi Na Muchi"], + "date" : "dt:2014-06-28 00:00:00", + "gallery_id" : 37387, + "lang" : "en", + "language" : "English", + "parodies" : ["Toaru Kagaku No Railgun"], + "relationships": list, + "tags" : list, + "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", + "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", + "type" : "Doujinshi", +}, + +{ + "#url" : "https://hentaihand.com/en/artist/takumi-na-muchi", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, + "#pattern" : hentaihand.HentaihandGalleryExtractor.pattern, + "#count" : ">= 6", +}, + +{ + "#url" : "https://hentaihand.com/en/tag/full-color", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +{ + "#url" : "https://hentaihand.com/fr/language/japanese", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +{ + "#url" : "https://hentaihand.com/zh/category/manga", + "#category": ("", "hentaihand", "tag"), + "#class" : hentaihand.HentaihandTagExtractor, +}, + +) diff --git a/test/results/hentaihere.py b/test/results/hentaihere.py new file mode 100644 index 00000000..abe52fe9 --- /dev/null +++ b/test/results/hentaihere.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hentaihere + + +__tests__ = ( +{ + "#url" : "https://hentaihere.com/m/S13812/1/1/", + "#category": ("", "hentaihere", "chapter"), + "#class" : hentaihere.HentaihereChapterExtractor, + "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", +}, + +{ + "#url" : "https://hentaihere.com/m/S23048/1.5/1/", + "#category": ("", "hentaihere", "chapter"), + "#class" : hentaihere.HentaihereChapterExtractor, + "#pattern" : r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", + "#count" : 32, + + "author" : "Shinozuka Yuuji", + "chapter" : 1, + "chapter_id" : 80186, + "chapter_minor": ".5", + "count" : 32, + "lang" : "en", + "language" : "English", + "manga" : "High School Slut's Love Consultation", + "manga_id" : 23048, + "page" : int, + "title" : "High School Slut's Love Consultation + Girlfriend [Full Color]", + "type" : "Original", +}, + +{ + "#url" : "https://hentaihere.com/m/S13812", + "#category": ("", "hentaihere", "manga"), + "#class" : hentaihere.HentaihereMangaExtractor, + "#sha1_url" : "d1ba6e28bb2162e844f8559c2b2725ba0a093559", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", +}, + +{ + "#url" : "https://hentaihere.com/m/S7608", + "#category": ("", "hentaihere", "manga"), + "#class" : hentaihere.HentaihereMangaExtractor, + "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": "", + "lang" : "en", + "language" : "English", + "manga" : "Oshikake Riot", + "manga_id" : 7608, + "title" : r"re:Oshikake Riot( \d+)?", + "type" : "Original", +}, + +) diff --git a/test/results/hiperdex.py b/test/results/hiperdex.py new file mode 100644 index 00000000..33948f22 --- /dev/null +++ b/test/results/hiperdex.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hiperdex + + +__tests__ = ( +{ + "#url" : "https://hiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, + "#pattern" : r"https://(1st)?hiperdex\d?.(com|net|info)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", + "#count" : 9, + + "artist" : "Sasuga Kei", + "author" : "Sasuga Kei", + "chapter" : 154, + "chapter_minor": ".5", + "description" : r"re:Natsuo Fujii is in love with his teacher, ", + "genre" : list, + "manga" : "Domestic na Kanojo", + "release" : 2014, + "score" : float, + "type" : "Manga", +}, + +{ + "#url" : "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class" : hiperdex.HiperdexChapterExtractor, +}, + +{ + "#url" : "https://hiperdex.com/manga/1603231576-youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, + "#pattern" : hiperdex.HiperdexChapterExtractor.pattern, + "#count" : 51, + + "artist" : "Bolp", + "author" : "Abyo4", + "chapter" : int, + "chapter_minor": "", + "description" : r"re:I didn’t think much of the creepy girl in ", + "genre" : list, + "manga" : "You’re Not That Special!", + "release" : 2019, + "score" : float, + "status" : "Completed", + "type" : "Manhwa", +}, + +{ + "#url" : "https://hiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://1sthiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class" : hiperdex.HiperdexMangaExtractor, +}, + +{ + "#url" : "https://1sthiperdex.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.net/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex2.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.info/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, +}, + +{ + "#url" : "https://hiperdex.com/manga-author/viagra/", + "#category": ("", "hiperdex", "artist"), + "#class" : hiperdex.HiperdexArtistExtractor, + "#pattern" : hiperdex.HiperdexMangaExtractor.pattern, + "#count" : ">= 6", +}, + +) diff --git a/test/results/hitomi.py b/test/results/hitomi.py new file mode 100644 index 00000000..10e94135 --- /dev/null +++ b/test/results/hitomi.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hitomi + + +__tests__ = ( +{ + "#url" : "https://hitomi.la/galleries/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#pattern" : r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", + "#count" : 16, + "#sha1_metadata": "86af5371f38117a07407f11af689bdd460b09710", +}, + +{ + "#url" : "https://hitomi.la/galleries/1401410.html", + "#comment" : "download test", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#range" : "1", + "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", +}, + +{ + "#url" : "https://hitomi.la/galleries/733697.html", + "#comment" : "Game CG with scenes (#321)", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#count" : 210, +}, + +{ + "#url" : "https://hitomi.la/galleries/1045954.html", + "#comment" : "fallback for galleries only available through /reader/ URLs", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#count" : 1413, +}, + +{ + "#url" : "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", + "#comment" : "gallery with 'broken' redirect", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#options" : {"format": "original"}, + "#pattern" : r"https://[a-c]b\.hitomi\.la/images/\d+/\d+/[0-9a-f]{64}\.jpg", + "#count" : 10, +}, + +{ + "#url" : "https://hitomi.la/cg/1615823.html", + "#comment" : "no tags", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, + "#options" : {"format": "avif"}, + "#pattern" : r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", + "#count" : 22, +}, + +{ + "#url" : "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/manga/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/doujinshi/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/cg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/gamecg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/reader/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class" : hitomi.HitomiGalleryExtractor, +}, + +{ + "#url" : "https://hitomi.la/tag/screenshots-japanese.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, + "#pattern" : hitomi.HitomiGalleryExtractor.pattern, + "#count" : ">= 35", +}, + +{ + "#url" : "https://hitomi.la/artist/a1-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/group/initial%2Dg-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/series/amnesia-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/type/doujinshi-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +{ + "#url" : "https://hitomi.la/character/a2-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class" : hitomi.HitomiTagExtractor, +}, + +) diff --git a/test/results/horne.py b/test/results/horne.py new file mode 100644 index 00000000..1cea3a0e --- /dev/null +++ b/test/results/horne.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nijie +import datetime + + +__tests__ = ( +{ + "#url" : "https://horne.red/members.php?id=58000", + "#category": ("Nijie", "horne", "user"), + "#class" : nijie.NijieUserExtractor, +}, + +{ + "#url" : "https://horne.red/members_illust.php?id=58000", + "#category": ("Nijie", "horne", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", + "#range" : "1-20", + "#count" : 20, + + "artist_id" : 58000, + "artist_name": "のえるわ", + "date" : datetime.datetime, + "description": str, + "image_id" : int, + "num" : int, + "tags" : list, + "title" : str, + "url" : str, + "user_id" : 58000, + "user_name" : "のえるわ", +}, + +{ + "#url" : "https://horne.red/members_dojin.php?id=58000", + "#category": ("Nijie", "horne", "doujin"), + "#class" : nijie.NijieDoujinExtractor, +}, + +{ + "#url" : "https://horne.red/user_like_illust_view.php?id=58000", + "#category": ("Nijie", "horne", "favorite"), + "#class" : nijie.NijieFavoriteExtractor, + "#range" : "1-5", + "#count" : 5, + + "user_id" : 58000, + "user_name": "のえるわ", +}, + +{ + "#url" : "https://horne.red/history_nuita.php?id=58000", + "#category": ("Nijie", "horne", "nuita"), + "#class" : nijie.NijieNuitaExtractor, +}, + +{ + "#url" : "https://horne.red/like_user_view.php", + "#category": ("Nijie", "horne", "feed"), + "#class" : nijie.NijieFeedExtractor, +}, + +{ + "#url" : "https://horne.red/like_my.php", + "#category": ("Nijie", "horne", "followed"), + "#class" : nijie.NijieFollowedExtractor, +}, + +{ + "#url" : "https://horne.red/view.php?id=8716", + "#category": ("Nijie", "horne", "image"), + "#class" : nijie.NijieImageExtractor, + "#count" : 4, + + "artist_id" : 58000, + "artist_name": "のえるわ", + "date" : "dt:2018-02-04 14:47:24", + "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", + "image_id" : 8716, + "tags" : [ + "男の娘", + "フェラ", + "オリキャラ", + "うちのこ", + ], + "title" : "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", + "user_id" : 58000, + "user_name" : "のえるわ", +}, + +) diff --git a/test/results/hotleak.py b/test/results/hotleak.py new file mode 100644 index 00000000..4e577d11 --- /dev/null +++ b/test/results/hotleak.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import hotleak +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://hotleak.vip/kaiyakawaii/photo/1617145", + "#category": ("", "hotleak", "post"), + "#class" : hotleak.HotleakPostExtractor, + "#pattern" : r"https://hotleak\.vip/storage/images/3625/1617145/fefdd5988dfcf6b98cc9e11616018868\.jpg", + + "id" : 1617145, + "creator" : "kaiyakawaii", + "type" : "photo", + "filename" : "fefdd5988dfcf6b98cc9e11616018868", + "extension": "jpg", +}, + +{ + "#url" : "https://hotleak.vip/lilmochidoll/video/1625538", + "#category": ("", "hotleak", "post"), + "#class" : hotleak.HotleakPostExtractor, + "#pattern" : r"ytdl:https://cdn8-leak\.camhdxx\.com/1661/1625538/index\.m3u8", + + "id" : 1625538, + "creator" : "lilmochidoll", + "type" : "video", + "filename" : "index", + "extension": "mp4", +}, + +{ + "#url" : "https://hotleak.vip/kaiyakawaii", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#range" : "1-200", + "#count" : 200, +}, + +{ + "#url" : "https://hotleak.vip/stellaviolet", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#count" : "> 600", +}, + +{ + "#url" : "https://hotleak.vip/doesnotexist", + "#category": ("", "hotleak", "creator"), + "#class" : hotleak.HotleakCreatorExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://hotleak.vip/photos", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, + "#pattern" : hotleak.HotleakPostExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://hotleak.vip/videos", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, +}, + +{ + "#url" : "https://hotleak.vip/creators", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, + "#pattern" : hotleak.HotleakCreatorExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://hotleak.vip/hot", + "#category": ("", "hotleak", "category"), + "#class" : hotleak.HotleakCategoryExtractor, +}, + +{ + "#url" : "https://hotleak.vip/search?search=gallery-dl", + "#category": ("", "hotleak", "search"), + "#class" : hotleak.HotleakSearchExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://hotleak.vip/search?search=hannah", + "#category": ("", "hotleak", "search"), + "#class" : hotleak.HotleakSearchExtractor, + "#count" : "> 30", +}, + +) diff --git a/test/results/hypnohub.py b/test/results/hypnohub.py new file mode 100644 index 00000000..d979b620 --- /dev/null +++ b/test/results/hypnohub.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", + "#category": ("gelbooru_v02", "hypnohub", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=pool&s=show&id=61", + "#category": ("gelbooru_v02", "hypnohub", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, + "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", + "#category": ("gelbooru_v02", "hypnohub", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://hypnohub.net/index.php?page=post&s=view&id=1439", + "#category": ("gelbooru_v02", "hypnohub", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : { + "tags" : True, + "notes": True, + }, + "#pattern" : r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", + "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", + + "tags_artist" : "brokenteapot", + "tags_character": "hsien-ko", + "tags_copyright": "capcom darkstalkers", + "tags_general" : str, + "tags_metadata" : "dialogue text translated", + "notes" : [ + { + "body" : "Master Master Master Master Master Master", + "height": 83, + "id" : 10577, + "width" : 129, + "x" : 259, + "y" : 20, + }, + { + "body" : "Response Response Response Response Response Response", + "height": 86, + "id" : 10578, + "width" : 125, + "x" : 126, + "y" : 20, + }, + { + "body" : "Obedience Obedience Obedience Obedience Obedience Obedience", + "height": 80, + "id" : 10579, + "width" : 98, + "x" : 20, + "y" : 20, + }, + ], +}, + +) diff --git a/test/results/idolcomplex.py b/test/results/idolcomplex.py new file mode 100644 index 00000000..6aced68c --- /dev/null +++ b/test/results/idolcomplex.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import idolcomplex + + +__tests__ = ( +{ + "#url" : "https://idol.sankakucomplex.com/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, + "#pattern" : r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", + "#range" : "18-22", + "#count" : 5, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/?tags=order:favcount", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, + "#range" : "18-22", + "#count" : 5, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", + "#category": ("booru", "idolcomplex", "tag"), + "#class" : idolcomplex.IdolcomplexTagExtractor, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/pool/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class" : idolcomplex.IdolcomplexPoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/post/show/694215", + "#category": ("booru", "idolcomplex", "post"), + "#class" : idolcomplex.IdolcomplexPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_idol" : str, + "tags_medium" : str, + "tags_general" : str, +}, + +) diff --git a/test/results/illusioncardsbooru.py b/test/results/illusioncardsbooru.py new file mode 100644 index 00000000..ba079364 --- /dev/null +++ b/test/results/illusioncardsbooru.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", + "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", + "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", + "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", + "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", +}, + +) diff --git a/test/results/imagebam.py b/test/results/imagebam.py new file mode 100644 index 00000000..759e2dc4 --- /dev/null +++ b/test/results/imagebam.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagebam +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#sha1_url" : "76d976788ae2757ac81694736b07b72356f5c4c8", + "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", + "#sha1_content" : "596e6bfa157f2c7169805d50075c2986549973a8", +}, + +{ + "#url" : "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#count" : 107, + "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", +}, + +{ + "#url" : "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://www.imagebam.com/view/GA3MT1", + "#comment" : "/view/ path (#2378)", + "#category": ("", "imagebam", "gallery"), + "#class" : imagebam.ImagebamGalleryExtractor, + "#sha1_url" : "35018ce1e00a2d2825a33d3cd37857edaf804919", + "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", +}, + +{ + "#url" : "https://www.imagebam.com/image/94d56c502511890", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url" : "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", + "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, +}, + +{ + "#url" : "https://www.imagebam.com/image/0850951366904951", + "#comment" : "NSFW (#1534)", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", +}, + +{ + "#url" : "https://www.imagebam.com/view/ME8JOQP", + "#comment" : "/view/ path (#2378)", + "#category": ("", "imagebam", "image"), + "#class" : imagebam.ImagebamImageExtractor, + "#sha1_url" : "4dca72bbe61a0360185cf4ab2bed8265b49565b8", + "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", + "#sha1_content" : "f81008666b17a42d8834c4749b910e1dc10a6e83", +}, + +) diff --git a/test/results/imagechest.py b/test/results/imagechest.py new file mode 100644 index 00000000..4b6a2ba3 --- /dev/null +++ b/test/results/imagechest.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagechest +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imgchest.com/p/3na7kr3by8d", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count" : 3, + "#sha1_url" : "7328ca4ec2459378d725e3be19f661d2b045feda", + "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", + + "count" : 3, + "gallery_id": "3na7kr3by8d", + "num" : int, + "title" : "Wizardry - Video Game From The Mid 80's", +}, + +{ + "#url" : "https://imgchest.com/p/9p4n3q2z7nq", + "#comment" : "'Load More Files' button (#4028)", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count" : 52, + "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", +}, + +{ + "#url" : "https://imgchest.com/p/xxxxxxxxxxx", + "#category": ("", "imagechest", "gallery"), + "#class" : imagechest.ImagechestGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/imagefap.py b/test/results/imagefap.py new file mode 100644 index 00000000..99a22166 --- /dev/null +++ b/test/results/imagefap.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagefap + + +__tests__ = ( +{ + "#url" : "https://www.imagefap.com/gallery/7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#sha1_metadata": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b", + "#sha1_content" : "694a0a57385980a6f90fbc296cadcd6c11ba2dab", +}, + +{ + "#url" : "https://www.imagefap.com/gallery/7876223", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#count" : 44, + + "categories" : [ + "Asses", + "Softcore", + "Pornstars", + ], + "count" : 44, + "description": "", + "gallery_id" : 7876223, + "image_id" : int, + "num" : int, + "tags" : [ + "big ass", + "panties", + "horny", + "pussy", + "exposed", + "outdoor", + ], + "title" : "Kelsi Monroe in lingerie", + "uploader" : "BdRachel", +}, + +{ + "#url" : "https://www.imagefap.com/gallery/6180555", + "#comment" : "description (#3905)", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, + "#range" : "1", + + "categories" : [ + "Amateur", + "Softcore", + "Homemade", + ], + "count" : 36, + "description": "Nude and dressed sluts showing off the goods", + "gallery_id" : 6180555, + "image_id" : int, + "num" : int, + "tags" : [], + "title" : "Dressed or Undressed MG*", + "uploader" : "splitopen", +}, + +{ + "#url" : "https://www.imagefap.com/pictures/7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://beta.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class" : imagefap.ImagefapGalleryExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class" : imagefap.ImagefapImageExtractor, + "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/65/196/1962981893\.jpg", + + "date" : "21/08/2014", + "gallery_id": 7876223, + "height" : 1600, + "image_id" : 1962981893, + "title" : "Kelsi Monroe in lingerie", + "uploader" : "BdRachel", + "width" : 1066, +}, + +{ + "#url" : "https://beta.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class" : imagefap.ImagefapImageExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/organizer/409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : r"https://www\.imagefap\.com/gallery/7876223", + "#count" : 1, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class" : imagefap.ImagefapFolderExtractor, + "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, + "#range" : "1-40", +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, + "#pattern" : imagefap.ImagefapFolderExtractor.pattern, + "#count" : ">= 18", +}, + +{ + "#url" : "https://www.imagefap.com/usergallery.php?userid=1862791", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, + "#pattern" : r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", + "#count" : 1, +}, + +{ + "#url" : "https://www.imagefap.com/profile/BdRachel/galleries", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +{ + "#url" : "https://www.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +{ + "#url" : "https://beta.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class" : imagefap.ImagefapUserExtractor, +}, + +) diff --git a/test/results/imagetwist.py b/test/results/imagetwist.py new file mode 100644 index 00000000..4606a773 --- /dev/null +++ b/test/results/imagetwist.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, + "#sha1_url" : "8d5e168c0bee30211f821c6f3b2116e419d42671", + "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +{ + "#url" : "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class" : imagehosts.ImagetwistImageExtractor, +}, + +) diff --git a/test/results/imagevenue.py b/test/results/imagevenue.py new file mode 100644 index 00000000..f57b1c94 --- /dev/null +++ b/test/results/imagevenue.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://www.imagevenue.com/ME13LS07", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#pattern" : r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", + "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", + "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", +}, + +{ + "#url" : "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", +}, + +{ + "#url" : "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", + "#category": ("imagehost", "imagevenue", "image"), + "#class" : imagehosts.ImagevenueImageExtractor, + "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", +}, + +) diff --git a/test/results/imgbb.py b/test/results/imgbb.py new file mode 100644 index 00000000..adc1dcf9 --- /dev/null +++ b/test/results/imgbb.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgbb +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://ibb.co/album/i5PggF", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#range" : "1-80", + "#sha1_url" : "70afec9fcc3a6de62a6b644b487d892d8d47cf1a", + "#sha1_metadata": "569e1d88ebdd27655387559cdf1cd526a3e1ab69", +}, + +{ + "#url" : "https://ibb.co/album/i5PggF?sort=title_asc", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#range" : "1-80", + "#sha1_url" : "afdf5fc95d8e09d77e8f44312f3e9b843987bb5a", + "#sha1_metadata": "f090e14d0e5f7868595082b2c95da1309c84872d", +}, + +{ + "#url" : "https://ibb.co/album/kYKpwF", + "#comment" : "no user data (#471)", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", + + "user": "", +}, + +{ + "#url" : "https://ibb.co/album/hqgWrF", + "#comment" : "private", + "#category": ("", "imgbb", "album"), + "#class" : imgbb.ImgbbAlbumExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://folkie.imgbb.com", + "#category": ("", "imgbb", "user"), + "#class" : imgbb.ImgbbUserExtractor, + "#pattern" : r"https?://i\.ibb\.co/\w+/[^/?#]+", + "#range" : "1-80", +}, + +{ + "#url" : "https://ibb.co/fUqh5b", + "#category": ("", "imgbb", "image"), + "#class" : imgbb.ImgbbImageExtractor, + "#pattern" : r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", + "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", + + "id" : "fUqh5b", + "title" : "Arundel Ireeman 5", + "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", + "width" : 960, + "height" : 719, + "user" : "folkie", + "extension": "jpg", +}, + +) diff --git a/test/results/imgbox.py b/test/results/imgbox.py new file mode 100644 index 00000000..ed0be593 --- /dev/null +++ b/test/results/imgbox.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgbox +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imgbox.com/g/JaX5V5HX7g", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#sha1_url" : "da4f15b161461119ee78841d4b8e8d054d95f906", + "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", + "#sha1_content" : "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", +}, + +{ + "#url" : "https://imgbox.com/g/cUGEkRbdZZ", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#sha1_url" : "76506a3aab175c456910851f66227e90484ca9f7", + "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", +}, + +{ + "#url" : "https://imgbox.com/g/JaX5V5HX7h", + "#category": ("", "imgbox", "gallery"), + "#class" : imgbox.ImgboxGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://imgbox.com/qHhw7lpG", + "#category": ("", "imgbox", "image"), + "#class" : imgbox.ImgboxImageExtractor, + "#sha1_url" : "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", + "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://imgbox.com/qHhw7lpH", + "#category": ("", "imgbox", "image"), + "#class" : imgbox.ImgboxImageExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/imgclick.py b/test/results/imgclick.py new file mode 100644 index 00000000..983f21f4 --- /dev/null +++ b/test/results/imgclick.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", + "#category": ("imagehost", "imgclick", "image"), + "#class" : imagehosts.ImgclickImageExtractor, + "#sha1_url" : "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", + "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/imgspice.py b/test/results/imgspice.py new file mode 100644 index 00000000..b2691b6a --- /dev/null +++ b/test/results/imgspice.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://imgspice.com/nwfwtpyog50y/test.png.html", + "#category": ("imagehost", "imgspice", "image"), + "#class" : imagehosts.ImgspiceImageExtractor, + "#sha1_url" : "b8c30a8f51ee1012959a4cfd46197fabf14de984", + "#sha1_metadata": "100e310a19a2fa22d87e1bbc427ecb9f6501e0c0", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +) diff --git a/test/results/imgth.py b/test/results/imgth.py new file mode 100644 index 00000000..333e1352 --- /dev/null +++ b/test/results/imgth.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgth + + +__tests__ = ( +{ + "#url" : "https://imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class" : imgth.ImgthGalleryExtractor, + "#pattern" : r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", + "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", + + "count" : 12, + "date" : "dt:2009-11-25 18:21:00", + "extension" : "jpg", + "filename" : r"re:wallpaper-anime_\w+", + "gallery_id": 37, + "num" : int, + "title" : "Wallpaper anime", + "user" : "celebrities", +}, + +{ + "#url" : "https://www.imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class" : imgth.ImgthGalleryExtractor, +}, + +) diff --git a/test/results/imgur.py b/test/results/imgur.py new file mode 100644 index 00000000..4fedc600 --- /dev/null +++ b/test/results/imgur.py @@ -0,0 +1,367 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imgur +from gallery_dl import exception +import datetime + + +__tests__ = ( +{ + "#url" : "https://imgur.com/21yMxCS", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#sha1_url" : "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "account_id" : 0, + "comment_count" : int, + "cover_id" : "21yMxCS", + "date" : "dt:2016-11-10 14:24:35", + "description" : "", + "downvote_count": int, + "duration" : 0, + "ext" : "png", + "favorite" : False, + "favorite_count": 0, + "has_sound" : False, + "height" : 32, + "id" : "21yMxCS", + "image_count" : 1, + "in_most_viral" : False, + "is_ad" : False, + "is_album" : False, + "is_animated" : False, + "is_looping" : False, + "is_mature" : False, + "is_pending" : False, + "mime_type" : "image/png", + "name" : "test-テスト", + "point_count" : int, + "privacy" : "", + "score" : int, + "size" : 182, + "title" : "Test", + "upvote_count" : int, + "url" : "https://i.imgur.com/21yMxCS.png", + "view_count" : int, + "width" : 64, +}, + +{ + "#url" : "http://imgur.com/0gybAXR", + "#comment" : "gifv/mp4 video", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#sha1_url" : "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", + "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", +}, + +{ + "#url" : "https://imgur.com/XFfsmuC", + "#comment" : "missing title in API response (#467)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + + "title": "Tears are a natural response to irritants", +}, + +{ + "#url" : "https://imgur.com/1Nily2P", + "#comment" : "animated png", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#pattern" : "https://i.imgur.com/1Nily2P.png", +}, + +{ + "#url" : "https://imgur.com/zzzzzzz", + "#comment" : "not found", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://m.imgur.com/r/Celebs/iHJ7tsM", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://www.imgur.com/21yMxCS", + "#comment" : "www", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://m.imgur.com/21yMxCS", + "#comment" : "mobile", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.com/zxaY6", + "#comment" : "5 character key", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.io/zxaY6", + "#comment" : ".io", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/21yMxCS.png", + "#comment" : "direct link", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.io/21yMxCS.png", + "#comment" : "direct link .io", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/21yMxCSh.png", + "#comment" : "direct link thumbnail", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/zxaY6.gif", + "#comment" : "direct link (short)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://i.imgur.com/zxaY6s.gif", + "#comment" : "direct link (short; thumb)", + "#category": ("", "imgur", "image"), + "#class" : imgur.ImgurImageExtractor, +}, + +{ + "#url" : "https://imgur.com/a/TcBmP", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", + + "album" : { + "account_id" : 0, + "comment_count" : int, + "cover_id" : "693j2Kr", + "date" : "dt:2015-10-09 10:37:50", + "description" : "", + "downvote_count": 0, + "favorite" : False, + "favorite_count": 0, + "id" : "TcBmP", + "image_count" : 19, + "in_most_viral" : False, + "is_ad" : False, + "is_album" : True, + "is_mature" : False, + "is_pending" : False, + "privacy" : "private", + "score" : int, + "title" : "138", + "upvote_count" : int, + "url" : "https://imgur.com/a/TcBmP", + "view_count" : int, + "virality" : int, + }, + "account_id" : 0, + "count" : 19, + "date" : datetime.datetime, + "description": "", + "ext" : "jpg", + "has_sound" : False, + "height" : int, + "id" : str, + "is_animated": False, + "is_looping" : False, + "mime_type" : "image/jpeg", + "name" : str, + "num" : int, + "size" : int, + "title" : str, + "type" : "image", + "updated_at" : None, + "url" : str, + "width" : int, +}, + +{ + "#url" : "https://imgur.com/a/eD9CT", + "#comment" : "large album", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "de748c181a04d18bef1de9d4f4866ef0a06d632b", +}, + +{ + "#url" : "https://imgur.com/a/RhJXhVT/all", + "#comment" : "7 character album hash", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", +}, + +{ + "#url" : "https://imgur.com/a/TcBmQ", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, +}, + +{ + "#url" : "https://imgur.com/a/pjOnJA0", + "#comment" : "empty, no 'media' (#2557)", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.imgur.com/a/TcBmP", + "#comment" : "www", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://imgur.io/a/TcBmP", + "#comment" : ".io", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://m.imgur.com/a/TcBmP", + "#comment" : "mobile", + "#category": ("", "imgur", "album"), + "#class" : imgur.ImgurAlbumExtractor, +}, + +{ + "#url" : "https://imgur.com/gallery/zf2fIms", + "#comment" : "non-album gallery (#380)", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, + "#pattern" : "https://imgur.com/zf2fIms", +}, + +{ + "#url" : "https://imgur.com/gallery/eD9CT", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, + "#pattern" : "https://imgur.com/a/eD9CT", +}, + +{ + "#url" : "https://imgur.com/t/unmuted/26sEhNr", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.com/t/cat/qSB8NbN", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.io/t/cat/qSB8NbN", + "#comment" : ".io", + "#category": ("", "imgur", "gallery"), + "#class" : imgur.ImgurGalleryExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/posts", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/submitted", + "#category": ("", "imgur", "user"), + "#class" : imgur.ImgurUserExtractor, +}, + +{ + "#url" : "https://imgur.com/user/Miguenzo/favorites", + "#category": ("", "imgur", "favorite"), + "#class" : imgur.ImgurFavoriteExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896757/public", + "#category": ("", "imgur", "favorite-folder"), + "#class" : imgur.ImgurFavoriteFolderExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#count" : 3, +}, + +{ + "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896741/private", + "#category": ("", "imgur", "favorite-folder"), + "#class" : imgur.ImgurFavoriteFolderExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#count" : 5, +}, + +{ + "#url" : "https://imgur.com/r/pics", + "#category": ("", "imgur", "subreddit"), + "#class" : imgur.ImgurSubredditExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/t/animals", + "#category": ("", "imgur", "tag"), + "#class" : imgur.ImgurTagExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://imgur.com/search?q=cute+cat", + "#category": ("", "imgur", "search"), + "#class" : imgur.ImgurSearchExtractor, + "#pattern" : r"https://imgur\.com(/a)?/\w+$", + "#range" : "1-100", + "#count" : 100, +}, + +) diff --git a/test/results/imxto.py b/test/results/imxto.py new file mode 100644 index 00000000..430b388b --- /dev/null +++ b/test/results/imxto.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://imx.to/i/1qdeva", + "#comment" : "new-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url" : "ab2173088a6cdef631d7a47dec4a5da1c6a00130", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "size" : 18, + "width" : 64, + "height": 32, + "hash" : "94d56c599223c59f3feb71ea603484d1", +}, + +{ + "#url" : "https://imx.to/img-57a2050547b97.html", + "#comment" : "old-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url" : "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", + + "size" : 5284, + "width" : 320, + "height": 160, + "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", +}, + +{ + "#url" : "https://img.yt/img-57a2050547b97.html", + "#comment" : "img.yt domain", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", +}, + +{ + "#url" : "https://imx.to/img-57a2050547b98.html", + "#category": ("imagehost", "imxto", "image"), + "#class" : imagehosts.ImxtoImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://imx.to/g/ozdy", + "#category": ("imagehost", "imxto", "gallery"), + "#class" : imagehosts.ImxtoGalleryExtractor, + "#pattern" : imagehosts.ImxtoImageExtractor.pattern, + "#count" : 40, + + "title": "untitled gallery", +}, + +) diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py new file mode 100644 index 00000000..7129877e --- /dev/null +++ b/test/results/inkbunny.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import inkbunny +import datetime + + +__tests__ = ( +{ + "#url" : "https://inkbunny.net/soina", + "#category": ("", "inkbunny", "user"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", + "#range" : "20-50", + + "date" : datetime.datetime, + "deleted" : bool, + "file_id" : r"re:[0-9]+", + "filename" : r"re:[0-9]+_soina_\w+", + "full_file_md5" : r"re:[0-9a-f]{32}", + "mimetype" : str, + "submission_id" : r"re:[0-9]+", + "user_id" : "20969", + "comments_count" : r"re:[0-9]+", + "favorite" : bool, + "favorites_count": r"re:[0-9]+", + "friends_only" : bool, + "guest_block" : bool, + "hidden" : bool, + "pagecount" : r"re:[0-9]+", + "pools" : list, + "pools_count" : int, + "public" : bool, + "rating_id" : r"re:[0-9]+", + "rating_name" : str, + "ratings" : list, + "scraps" : bool, + "tags" : list, + "title" : str, + "type_name" : str, + "username" : "soina", + "views" : str, +}, + +{ + "#url" : "https://inkbunny.net/gallery/soina", + "#category": ("", "inkbunny", "gallery"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#range" : "1-25", + + "scraps": False, +}, + +{ + "#url" : "https://inkbunny.net/scraps/soina", + "#category": ("", "inkbunny", "scraps"), + "#class" : inkbunny.InkbunnyUserExtractor, + "#range" : "1-25", + + "scraps": True, +}, + +{ + "#url" : "https://inkbunny.net/poolview_process.php?pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class" : inkbunny.InkbunnyPoolExtractor, + "#count" : 9, + + "pool_id": "28985", +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", + "#category": ("", "inkbunny", "pool"), + "#class" : inkbunny.InkbunnyPoolExtractor, +}, + +{ + "#url" : "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class" : inkbunny.InkbunnyFavoriteExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", + "#range" : "20-50", + + "favs_user_id": "20969", +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class" : inkbunny.InkbunnyFavoriteExtractor, +}, + +{ + "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", + "#category": ("", "inkbunny", "search"), + "#class" : inkbunny.InkbunnySearchExtractor, + "#range" : "1-10", + "#count" : 10, + + "search": { + "rid" : "ffffffffff", + "mode" : "search", + "page" : "1", + "orderby" : "create_datetime", + "text" : "cute", + "stringtype" : "and", + "keywords" : "yes", + "title" : "yes", + "description": "no", + }, +}, + +{ + "#url" : "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class" : inkbunny.InkbunnyFollowingExtractor, + "#pattern" : inkbunny.InkbunnyUserExtractor.pattern, + "#count" : ">= 90", +}, + +{ + "#url" : "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", + "#category": ("", "inkbunny", "following"), + "#class" : inkbunny.InkbunnyFollowingExtractor, +}, + +{ + "#url" : "https://inkbunny.net/s/1829715", + "#category": ("", "inkbunny", "post"), + "#class" : inkbunny.InkbunnyPostExtractor, + "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", + "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", +}, + +{ + "#url" : "https://inkbunny.net/s/2044094", + "#category": ("", "inkbunny", "post"), + "#class" : inkbunny.InkbunnyPostExtractor, + "#count" : 4, +}, + +) diff --git a/test/results/instagram.py b/test/results/instagram.py new file mode 100644 index 00000000..7f601426 --- /dev/null +++ b/test/results/instagram.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import instagram + + +__tests__ = ( +{ + "#url" : "https://www.instagram.com/instagram/", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/?hl=en", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/id:25025320/", + "#category": ("", "instagram", "user"), + "#class" : instagram.InstagramUserExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/posts/", + "#category": ("", "instagram", "posts"), + "#class" : instagram.InstagramPostsExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/reels/", + "#category": ("", "instagram", "reels"), + "#class" : instagram.InstagramReelsExtractor, + "#range" : "40-60", + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.instagram.com/instagram/tagged/", + "#category": ("", "instagram", "tagged"), + "#class" : instagram.InstagramTaggedExtractor, + "#range" : "1-16", + "#count" : ">= 16", + + "tagged_owner_id" : "25025320", + "tagged_username" : "instagram", + "tagged_full_name": "Instagram", +}, + +{ + "#url" : "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", + "#category": ("", "instagram", "guide"), + "#class" : instagram.InstagramGuideExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/", + "#category": ("", "instagram", "saved"), + "#class" : instagram.InstagramSavedExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/all-posts/", + "#category": ("", "instagram", "saved"), + "#class" : instagram.InstagramSavedExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/saved/collection_name/123456789/", + "#category": ("", "instagram", "collection"), + "#class" : instagram.InstagramCollectionExtractor, +}, + +{ + "#url" : "https://www.instagram.com/stories/instagram/", + "#category": ("", "instagram", "stories"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/stories/highlights/18042509488170095/", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://instagram.com/stories/geekmig/2724343156064789461", + "#category": ("", "instagram", "stories"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramStoriesExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/highlights", + "#category": ("", "instagram", "highlights"), + "#class" : instagram.InstagramHighlightsExtractor, +}, + +{ + "#url" : "https://www.instagram.com/instagram/following", + "#category": ("", "instagram", "following"), + "#class" : instagram.InstagramFollowingExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/explore/tags/instagram/", + "#category": ("", "instagram", "tag"), + "#class" : instagram.InstagramTagExtractor, + "#range" : "1-16", + "#count" : ">= 16", +}, + +{ + "#url" : "https://www.instagram.com/instagram/avatar", + "#category": ("", "instagram", "avatar"), + "#class" : instagram.InstagramAvatarExtractor, + "#pattern" : r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", +}, + +{ + "#url" : "https://www.instagram.com/p/BqvsDleB3lV/", + "#comment" : "GraphImage", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", + + "date" : "dt:2018-11-29 01:04:04", + "description" : str, + "height" : int, + "likes" : int, + "location_id" : "214424288", + "location_slug" : "hong-kong", + "location_url" : r"re:/explore/locations/214424288/hong-kong/", + "media_id" : "1922949326347663701", + "shortcode" : "BqvsDleB3lV", + "post_id" : "1922949326347663701", + "post_shortcode": "BqvsDleB3lV", + "post_url" : "https://www.instagram.com/p/BqvsDleB3lV/", + "tags" : ["#WHPsquares"], + "typename" : "GraphImage", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/p/BoHk1haB5tM/", + "#comment" : "GraphSidecar", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#count" : 5, + + "sidecar_media_id": "1875629777499953996", + "sidecar_shortcode": "BoHk1haB5tM", + "post_id" : "1875629777499953996", + "post_shortcode" : "BoHk1haB5tM", + "post_url" : "https://www.instagram.com/p/BoHk1haB5tM/", + "num" : int, + "likes" : int, + "username" : "instagram", +}, + +{ + "#url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", + "#comment" : "GraphVideo", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"/46840863_726311431074534_7805566102611403091_n\.mp4", + + "date" : "dt:2018-11-29 19:23:58", + "description": str, + "height" : int, + "likes" : int, + "media_id" : "1923502432034620000", + "post_url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", + "shortcode" : "Bqxp0VSBgJg", + "tags" : ["#ASMR"], + "typename" : "GraphVideo", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/tv/BkQjCfsBIzi/", + "#comment" : "GraphVideo (IGTV)", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#pattern" : r"/10000000_597132547321814_702169244961988209_n\.mp4", + + "date" : "dt:2018-06-20 19:51:32", + "description": str, + "height" : int, + "likes" : int, + "media_id" : "1806097553666903266", + "post_url" : "https://www.instagram.com/p/BkQjCfsBIzi/", + "shortcode" : "BkQjCfsBIzi", + "typename" : "GraphVideo", + "username" : "instagram", + "width" : int, +}, + +{ + "#url" : "https://www.instagram.com/p/BtOvDOfhvRr/", + "#comment" : "GraphSidecar with 2 embedded GraphVideo objects", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + "#count" : 2, + + "post_url" : "https://www.instagram.com/p/BtOvDOfhvRr/", + "sidecar_media_id": "1967717017113261163", + "sidecar_shortcode": "BtOvDOfhvRr", + "video_url" : str, +}, + +{ + "#url" : "https://www.instagram.com/p/B_2lf3qAd3y/", + "#comment" : "GraphImage with tagged user", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, + + "tagged_users": [{ + "id" : "1246468638", + "username" : "kaaymbl", + "full_name": "Call Me Kay", +}], +}, + +{ + "#url" : "https://www.instagram.com/dm/p/CW042g7B9CY/", + "#comment" : "URL with username (#2085)", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, +}, + +{ + "#url" : "https://www.instagram.com/reel/CDg_6Y1pxWu/", + "#category": ("", "instagram", "post"), + "#class" : instagram.InstagramPostExtractor, +}, + +) diff --git a/test/results/issuu.py b/test/results/issuu.py new file mode 100644 index 00000000..5e086d1e --- /dev/null +++ b/test/results/issuu.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import issuu + + +__tests__ = ( +{ + "#url" : "https://issuu.com/issuu/docs/motions-1-2019/", + "#category": ("", "issuu", "publication"), + "#class" : issuu.IssuuPublicationExtractor, + "#pattern" : r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", + "#count" : 36, + + "document" : { + "access" : "PUBLIC", + "contentRating": { + "isAdsafe" : True, + "isExplicit": False, + "isReviewed": True, + }, + "date" : "dt:2019-09-16 00:00:00", + "description" : r"re:Motions, the brand new publication by I", + "documentName" : "motions-1-2019", + "downloadable" : False, + "pageCount" : 36, + "publicationId": "d99ec95935f15091b040cb8060f05510", + "title" : "Motions by Issuu - Issue 1", + "username" : "issuu", + }, + "extension": "jpg", + "filename" : r"re:page_\d+", + "num" : int, +}, + +{ + "#url" : "https://issuu.com/issuu", + "#category": ("", "issuu", "user"), + "#class" : issuu.IssuuUserExtractor, + "#pattern" : issuu.IssuuPublicationExtractor.pattern, + "#count" : "> 25", +}, + +) diff --git a/test/results/itaku.py b/test/results/itaku.py new file mode 100644 index 00000000..ef3d3679 --- /dev/null +++ b/test/results/itaku.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import itaku + + +__tests__ = ( +{ + "#url" : "https://itaku.ee/profile/piku/gallery", + "#category": ("", "itaku", "gallery"), + "#class" : itaku.ItakuGalleryExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/[^/?#]+\.(jpg|png|gif)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://itaku.ee/images/100471", + "#category": ("", "itaku", "image"), + "#class" : itaku.ItakuImageExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/220504_oUNIAFT\.png", + "#count" : 1, + + "already_pinned" : None, + "blacklisted" : { + "blacklisted_tags": [], + "is_blacklisted" : False, + }, + "can_reshare" : True, + "date" : "dt:2022-05-05 19:21:17", + "date_added" : "2022-05-05T19:21:17.674148Z", + "date_edited" : "2022-05-25T14:37:46.220612Z", + "description" : "sketch from drawpile", + "extension" : "png", + "filename" : "220504_oUNIAFT", + "hotness_score" : float, + "id" : 100471, + "image" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT.png", + "image_xl" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT/lg.jpg", + "liked_by_you" : False, + "maturity_rating" : "SFW", + "num_comments" : int, + "num_likes" : int, + "num_reshares" : int, + "obj_tags" : 136446, + "owner" : 16775, + "owner_avatar" : "https://d1wmr8tlk3viaj.cloudfront.net/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_displayname": "Piku", + "owner_username" : "piku", + "reshared_by_you" : False, + "sections" : ["Fanart/Miku"], + "tags" : list, + "tags_character" : ["hatsune_miku"], + "tags_copyright" : ["vocaloid"], + "tags_general" : [ + "twintails", + "green_hair", + "flag", + "gloves", + "green_eyes", + "female", + "racing_miku", + ], + "title" : "Racing Miku 2022 Ver.", + "too_mature" : False, + "uncompressed_filesize": "0.62", + "video" : None, + "visibility" : "PUBLIC", +}, + +{ + "#url" : "https://itaku.ee/images/19465", + "#comment" : "video", + "#category": ("", "itaku", "image"), + "#class" : itaku.ItakuImageExtractor, + "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids/sleepy_af_OY5GHWw\.mp4", +}, + +) diff --git a/test/results/itchio.py b/test/results/itchio.py new file mode 100644 index 00000000..d2acfa61 --- /dev/null +++ b/test/results/itchio.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import itchio + + +__tests__ = ( +{ + "#url" : "https://sirtartarus.itch.io/a-craft-of-mine", + "#category": ("", "itchio", "game"), + "#class" : itchio.ItchioGameExtractor, + "#pattern" : r"https://\w+\.ssl\.hwcdn\.net/upload2/game/1983311/7723751\?", + "#count" : 1, + + "extension": "", + "filename" : "7723751", + "game" : { + "id" : 1983311, + "noun" : "game", + "title": "A Craft Of Mine", + "url" : "https://sirtartarus.itch.io/a-craft-of-mine", + }, + "user" : { + "id" : 4060052, + "name": "SirTartarus", + "url" : "https://sirtartarus.itch.io", + }, +}, + +) diff --git a/test/results/joyreactor.py b/test/results/joyreactor.py new file mode 100644 index 00000000..4c29c213 --- /dev/null +++ b/test/results/joyreactor.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://joyreactor.cc/tag/Advent+Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "http://joyreactor.com/tag/Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", +}, + +{ + "#url" : "http://joyreactor.com/tag/Dark+Souls+2/best", + "#comment" : "'best' rating (#3073)", + "#category": ("reactor", "joyreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#count" : 4, +}, + +{ + "#url" : "http://joyreactor.cc/search/Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://joyreactor.com/search?q=Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://joyreactor.cc/user/hemantic", + "#category": ("reactor", "joyreactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://joyreactor.com/user/Tacoman123", + "#category": ("reactor", "joyreactor", "user"), + "#class" : reactor.ReactorUserExtractor, + "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", +}, + +{ + "#url" : "http://joyreactor.com/post/3721876", + "#comment" : "single image", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", + "#count" : 1, + "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", +}, + +{ + "#url" : "http://joyreactor.com/post/3713804", + "#comment" : "4 images", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", + "#count" : 4, + "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", +}, + +{ + "#url" : "http://joyreactor.com/post/3726210", + "#comment" : "gif / video", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", + "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", +}, + +{ + "#url" : "http://joyreactor.com/post/3668724", + "#comment" : "youtube embed", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "bf1666eddcff10c9b58f6be63fa94e4e13074214", + "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", +}, + +{ + "#url" : "http://joyreactor.cc/post/1299", + "#comment" : "'malformed' JSON", + "#category": ("reactor", "joyreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", +}, + +) diff --git a/test/results/jpgfish.py b/test/results/jpgfish.py new file mode 100644 index 00000000..5aa4a126 --- /dev/null +++ b/test/results/jpgfish.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import jpgfish + + +__tests__ = ( +{ + "#url" : "https://jpg1.su/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, + "#pattern" : r"https://simp3\.jpg\.church/images/funnymeme\.jpg", + "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", + + "album" : "", + "extension": "jpg", + "filename" : "funnymeme", + "id" : "LecXGS", + "url" : "https://simp3.jpg.church/images/funnymeme.jpg", + "user" : "exearco", +}, + +{ + "#url" : "https://jpg.church/img/auCruA", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, + "#pattern" : r"https://simp2\.jpg\.church/hannahowo_00457\.jpg", + + "album": "401-500", +}, + +{ + "#url" : "https://jpeg.pet/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.pet/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.fishing/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.fish/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg.church/img/funnymeme.LecXGS", + "#category": ("", "jpgfish", "image"), + "#class" : jpgfish.JpgfishImageExtractor, +}, + +{ + "#url" : "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://jpg.fishing/a/gunggingnsk.N9OOI", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 114, +}, + +{ + "#url" : "https://jpg.fish/a/101-200.aNJ6A/", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 100, +}, + +{ + "#url" : "https://jpg.church/a/hannahowo.aNTdH/sub", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, + "#count" : 606, +}, + +{ + "#url" : "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, +}, + +{ + "#url" : "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("", "jpgfish", "album"), + "#class" : jpgfish.JpgfishAlbumExtractor, +}, + +{ + "#url" : "https://jpg1.su/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://jpg.church/exearco/albums", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://jpeg.pet/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.pet/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.fishing/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.fish/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +{ + "#url" : "https://jpg.church/exearco", + "#category": ("", "jpgfish", "user"), + "#class" : jpgfish.JpgfishUserExtractor, +}, + +) diff --git a/test/results/kabeuchi.py b/test/results/kabeuchi.py new file mode 100644 index 00000000..37b6a3d8 --- /dev/null +++ b/test/results/kabeuchi.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kabeuchi +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://kabe-uchiroom.com/mypage/?id=919865303848255493", + "#category": ("", "kabeuchi", "user"), + "#class" : kabeuchi.KabeuchiUserExtractor, + "#pattern" : r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", + "#count" : ">= 24", +}, + +{ + "#url" : "https://kabe-uchiroom.com/mypage/?id=123456789", + "#category": ("", "kabeuchi", "user"), + "#class" : kabeuchi.KabeuchiUserExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/keenspot.py b/test/results/keenspot.py new file mode 100644 index 00000000..9f756ad8 --- /dev/null +++ b/test/results/keenspot.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import keenspot + + +__tests__ = ( +{ + "#url" : "http://marksmen.keenspot.com/", + "#comment" : "link", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", +}, + +{ + "#url" : "http://barkercomic.keenspot.com/", + "#comment" : "id", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", +}, + +{ + "#url" : "http://crowscare.keenspot.com/", + "#comment" : "id v2", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", +}, + +{ + "#url" : "http://supernovas.keenspot.com/", + "#comment" : "ks", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", +}, + +{ + "#url" : "http://twokinds.keenspot.com/comic/1066/", + "#category": ("", "keenspot", "comic"), + "#class" : keenspot.KeenspotComicExtractor, + "#range" : "1-3", + "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", +}, + +) diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py new file mode 100644 index 00000000..f5419f02 --- /dev/null +++ b/test/results/kemonoparty.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import kemonoparty + + +__tests__ = ( +{ + "#url" : "https://kemono.party/fanbox/user/6993449", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyUserExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://kemono.party/patreon/user/881792?o=150", + "#comment" : "'max-posts' option, 'o' query parameter (#1674)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyUserExtractor, + "#options" : {"max-posts": 25}, + "#count" : "< 100", +}, + +{ + "#url" : "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyUserExtractor, +}, + +{ + "#url" : "https://kemono.party/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyUserExtractor, +}, + +{ + "#url" : "https://kemono.party/fanbox/user/6993449/post/506575", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono.party/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", + "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", + + "added" : "Wed, 06 May 2020 20:28:02 GMT", + "content" : str, + "count" : 1, + "date" : "dt:2019-08-11 02:09:04", + "edited" : None, + "embed" : dict, + "extension" : "jpeg", + "filename" : "P058kDFYus7DbqAkGlfWTlOr", + "hash" : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", + "id" : "506575", + "num" : 1, + "published" : "Sun, 11 Aug 2019 02:09:04 GMT", + "service" : "fanbox", + "shared_file": False, + "subcategory": "fanbox", + "title" : "c96取り置き", + "type" : "file", + "user" : "6993449", +}, + +{ + "#url" : "https://kemono.party/fanbox/user/7356311/post/802343", + "#comment" : "inline image (#1286)", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono\.party/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", + + "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", +}, + +{ + "#url" : "https://kemono.party/gumroad/user/trylsc/post/IURjT", + "#comment" : "kemono.party -> data.kemono.party", + "#category": ("", "kemonoparty", "gumroad"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"https://kemono\.party/data/(a4/7b/a47bfe938d8c1682eef06e885927484cd8df1b.+\.jpg|c6/04/c6048f5067fd9dbfa7a8be565ac194efdfb6e4.+\.zip)", +}, + +{ + "#url" : "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", + "#comment" : "username (#1548, #1652)", + "#category": ("", "kemonoparty", "gumroad"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"metadata": True}, + + "username": "Kudalyn's Creations", +}, + +{ + "#url" : "https://kemono.party/patreon/user/4158582/post/32099982", + "#comment" : "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://kemono.party/patreon/user/4158582/post/32099982", + "#comment" : "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"duplicates": True}, + "#count" : 3, +}, + +{ + "#url" : "https://kemono.party/patreon/user/34134344/post/38129255", + "#comment" : "DMs (#2008)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#options" : {"dms": True}, + + "dms": [{ + "body" : r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", + "date" : "2021-07-31 02:47:51.327865", +}], +}, + +{ + "#url" : "https://kemono.party/patreon/user/19623797/post/29035449", + "#comment" : "invalid file (#3510)", + "#category": ("", "kemonoparty", "patreon"), + "#class" : kemonoparty.KemonopartyPostExtractor, + "#pattern" : r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", + "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", +}, + +{ + "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://www.kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://beta.kemono.party/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class" : kemonoparty.KemonopartyPostExtractor, +}, + +{ + "#url" : "https://kemono.party/discord/server/488668827274444803#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#count" : 4, + + "channel_name": "finish-work", +}, + +{ + "#url" : "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#pattern" : r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", + "#count" : ">= 2", + + "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", +}, + +{ + "#url" : "https://kemono.party/discord/server/315262215055736843/channel/315262215055736843#general", + "#comment" : "'inline' files", + "#category": ("", "kemonoparty", "discord"), + "#class" : kemonoparty.KemonopartyDiscordExtractor, + "#options" : {"image-filter": "type == 'inline'"}, + "#pattern" : r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", + "#range" : "1-5", + + "hash": "", +}, + +{ + "#url" : "https://kemono.party/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class" : kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count" : 13, +}, + +{ + "#url" : "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class" : kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count" : 13, +}, + +{ + "#url" : "https://kemono.party/favorites", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, + "#count" : 3, + "#sha1_url": "f4b5b796979bcba824af84206578c79101c7f0e1", +}, + +{ + "#url" : "https://kemono.party/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, + "#count" : 3, + "#sha1_url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f", +}, + +{ + "#url" : "https://kemono.su/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class" : kemonoparty.KemonopartyFavoriteExtractor, + "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, + "#count" : 3, + "#sha1_url": "4be8e84cb384a907a8e7997baaf6287b451783b5", +}, + +) diff --git a/test/results/khinsider.py b/test/results/khinsider.py new file mode 100644 index 00000000..b680cab2 --- /dev/null +++ b/test/results/khinsider.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import khinsider + + +__tests__ = ( +{ + "#url" : "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", + "#category": ("", "khinsider", "soundtrack"), + "#class" : khinsider.KhinsiderSoundtrackExtractor, + "#pattern" : r"https?://vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", + "#count" : 1, + + "album" : { + "count" : 1, + "date" : "Sep 18th, 2016", + "name" : "Horizon Riders", + "platform": "Wii", + "size" : 26214400, + "type" : "Gamerip", + }, + "extension": "mp3", + "filename" : "Horizon Riders Wii - Full Soundtrack", +}, + +) diff --git a/test/results/kohlchan.py b/test/results/kohlchan.py new file mode 100644 index 00000000..479ed715 --- /dev/null +++ b/test/results/kohlchan.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lynxchan + + +__tests__ = ( +{ + "#url" : "https://kohlchan.net/a/res/4594.html", + "#category": ("lynxchan", "kohlchan", "thread"), + "#class" : lynxchan.LynxchanThreadExtractor, + "#pattern" : r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count" : ">= 80", +}, + +{ + "#url" : "https://kohlchan.net/a/", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, + "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://kohlchan.net/a/2.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +{ + "#url" : "https://kohlchan.net/a/catalog.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class" : lynxchan.LynxchanBoardExtractor, +}, + +) diff --git a/test/results/komikcast.py b/test/results/komikcast.py new file mode 100644 index 00000000..9a246009 --- /dev/null +++ b/test/results/komikcast.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import komikcast + + +__tests__ = ( +{ + "#url" : "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class" : komikcast.KomikcastChapterExtractor, + "#sha1_url" : "f6b43fbc027697749b3ea1c14931c83f878d7936", + "#sha1_metadata": "f3938e1aff9ad1f302f52447e9781b21f6da26d4", +}, + +{ + "#url" : "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class" : komikcast.KomikcastChapterExtractor, + "#sha1_url" : "efd00a9bd95461272d51990d7bc54b79ff3ff2e6", + "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", +}, + +{ + "#url" : "https://komikcast.site/komik/090-eko-to-issho/", + "#category": ("", "komikcast", "manga"), + "#class" : komikcast.KomikcastMangaExtractor, + "#sha1_url" : "19d3d50d532e84be6280a3d61ff0fd0ca04dd6b4", + "#sha1_metadata": "837a7e96867344ff59d840771c04c20dc46c0ab1", +}, + +{ + "#url" : "https://komikcast.me/tonari-no-kashiwagi-san/", + "#category": ("", "komikcast", "manga"), + "#class" : komikcast.KomikcastMangaExtractor, +}, + +) diff --git a/test/results/konachan.py b/test/results/konachan.py new file mode 100644 index 00000000..576acdaa --- /dev/null +++ b/test/results/konachan.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://konachan.com/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", + + "tags_artist" : "patata", + "tags_character": "clownpiece", + "tags_copyright": "touhou", + "tags_general" : str, +}, + +{ + "#url" : "https://konachan.net/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://konachan.com/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, + "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", +}, + +{ + "#url" : "https://konachan.com/post?tags=", + "#comment" : "empty 'tags' (#4354)", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://konachan.net/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://konachan.com/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, + "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", +}, + +{ + "#url" : "https://konachan.net/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://konachan.com/post/popular_by_month?month=11&year=2010", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, + "#count" : 20, +}, + +{ + "#url" : "https://konachan.com/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +{ + "#url" : "https://konachan.net/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/lensdump.py b/test/results/lensdump.py new file mode 100644 index 00000000..73f1fbdf --- /dev/null +++ b/test/results/lensdump.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lensdump + + +__tests__ = ( +{ + "#url" : "https://lensdump.com/a/1IhJr", + "#category": ("", "lensdump", "album"), + "#class" : lensdump.LensdumpAlbumExtractor, + "#pattern" : r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", + + "extension": "png", + "name" : str, + "num" : int, + "title" : str, + "url" : str, + "width" : int, +}, + +{ + "#url" : "https://lensdump.com/vstar925/albums", + "#category": ("", "lensdump", "albums"), + "#class" : lensdump.LensdumpAlbumsExtractor, +}, + +{ + "#url" : "https://lensdump.com/i/tyoAyM", + "#category": ("", "lensdump", "image"), + "#class" : lensdump.LensdumpImageExtractor, + "#pattern" : r"https://c\.l3n\.co/i/tyoAyM\.webp", + "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", + + "date" : "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename" : "tyoAyM", + "height" : 400, + "id" : "tyoAyM", + "title" : "MYOBI clovis bookcaseset", + "url" : "https://c.l3n.co/i/tyoAyM.webp", + "width" : 620, +}, + +) diff --git a/test/results/lesbianenergy.py b/test/results/lesbianenergy.py new file mode 100644 index 00000000..fffe8a0c --- /dev/null +++ b/test/results/lesbianenergy.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://lesbian.energy/@rerorero", + "#category": ("misskey", "lesbian.energy", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#pattern" : r"https://lesbian.energy/files/\w+", + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://lesbian.energy/@nano@mk.yopo.work", + "#category": ("misskey", "lesbian.energy", "user"), + "#class" : misskey.MisskeyUserExtractor, +}, + +{ + "#url" : "https://lesbian.energy/notes/995ig09wqy", + "#category": ("misskey", "lesbian.energy", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://lesbian.energy/notes/96ynd9w5kc", + "#category": ("misskey", "lesbian.energy", "note"), + "#class" : misskey.MisskeyNoteExtractor, +}, + +{ + "#url" : "https://lesbian.energy/my/favorites", + "#category": ("misskey", "lesbian.energy", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/lexica.py b/test/results/lexica.py new file mode 100644 index 00000000..07183f67 --- /dev/null +++ b/test/results/lexica.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lexica + + +__tests__ = ( +{ + "#url" : "https://lexica.art/?q=tree", + "#category": ("", "lexica", "search"), + "#class" : lexica.LexicaSearchExtractor, + "#pattern" : r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", + "#range" : "1-80", + "#count" : 80, + + "height" : int, + "id" : str, + "upscaled_height": int, + "upscaled_width" : int, + "userid" : str, + "width" : int, + "prompt" : { + "c" : int, + "grid" : bool, + "height" : int, + "id" : str, + "images" : list, + "initImage" : None, + "initImageStrength": None, + "model" : "lexica-aperture-v2", + "negativePrompt" : str, + "prompt" : str, + "seed" : str, + "timestamp" : r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", + "width" : int, + }, +}, + +) diff --git a/test/results/lightroom.py b/test/results/lightroom.py new file mode 100644 index 00000000..4c1a0532 --- /dev/null +++ b/test/results/lightroom.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lightroom + + +__tests__ = ( +{ + "#url" : "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", + "#category": ("", "lightroom", "gallery"), + "#class" : lightroom.LightroomGalleryExtractor, + "#count" : ">= 55", + + "title": "Sterne und Nachtphotos", + "user" : "Christian Schrang", +}, + +{ + "#url" : "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", + "#category": ("", "lightroom", "gallery"), + "#class" : lightroom.LightroomGalleryExtractor, + "#count" : ">= 180", + + "title": "HEBFC Snr/Res v Brighton", + "user" : "", +}, + +) diff --git a/test/results/livedoor.py b/test/results/livedoor.py new file mode 100644 index 00000000..7ab366f9 --- /dev/null +++ b/test/results/livedoor.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import livedoor + + +__tests__ = ( +{ + "#url" : "http://blog.livedoor.jp/zatsu_ke/", + "#category": ("", "livedoor", "blog"), + "#class" : livedoor.LivedoorBlogExtractor, + "#pattern" : r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", + "#range" : "1-50", + "#count" : 50, + "#archive" : False, + + "post" : { + "categories" : tuple, + "date" : "type:datetime", + "description": str, + "id" : int, + "tags" : list, + "title" : str, + "user" : "zatsu_ke", + }, + "filename": str, + "hash" : r"re:\w{4,}", + "num" : int, +}, + +{ + "#url" : "http://blog.livedoor.jp/uotapo/", + "#category": ("", "livedoor", "blog"), + "#class" : livedoor.LivedoorBlogExtractor, + "#range" : "1-5", + "#count" : 5, +}, + +{ + "#url" : "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", + "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", +}, + +{ + "#url" : "http://blog.livedoor.jp/amaumauma/archives/7835811.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "204bbd6a9db4969c50e0923855aeede04f2e4a62", + "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", +}, + +{ + "#url" : "http://blog.livedoor.jp/uotapo/archives/1050616939.html", + "#category": ("", "livedoor", "post"), + "#class" : livedoor.LivedoorPostExtractor, + "#sha1_url" : "4b5ab144b7309eb870d9c08f8853d1abee9946d2", + "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", +}, + +) diff --git a/test/results/lolibooru.py b/test/results/lolibooru.py new file mode 100644 index 00000000..f8750269 --- /dev/null +++ b/test/results/lolibooru.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://lolibooru.moe/post/show/281305/", + "#category": ("moebooru", "lolibooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"notes": True}, + "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", + + "notes": list, +}, + +{ + "#url" : "https://lolibooru.moe/post/show/287835", + "#category": ("moebooru", "lolibooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", + "#category": ("moebooru", "lolibooru", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/pool/show/239", + "#category": ("moebooru", "lolibooru", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://lolibooru.moe/post/popular_recent", + "#category": ("moebooru", "lolibooru", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/loudbooru.py b/test/results/loudbooru.py new file mode 100644 index 00000000..58236b7d --- /dev/null +++ b/test/results/loudbooru.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://loudbooru.com/post/list/original_character/1", + "#category": ("shimmie2", "loudbooru", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://loudbooru\.com/_images/[0-9a-f]{32}/\d+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://loudbooru.com/post/view/33828", + "#category": ("shimmie2", "loudbooru", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://loudbooru\.com/_images/.+\.png", + "#sha1_content": "a4755f787ba23ae2aa297a46810f802ca9032739", + + "extension": "png", + "file_url" : "https://loudbooru.com/_images/ca2638d903c86e8337fe9aeb4974be88/33828%20-%202020%20artist%3Astikyfinkaz%20character%3Alisa_loud%20cover%20fanfiction%3Aplatz_eins%20frowning%20half-closed_eyes%20solo%20text%20title_card.png", + "filename" : "33828 - 2020 artist:stikyfinkaz character:lisa_loud cover fanfiction:platz_eins frowning half-closed_eyes solo text title_card", + "height" : 1920, + "id" : 33828, + "md5" : "ca2638d903c86e8337fe9aeb4974be88", + "tags" : "2020 artist:stikyfinkaz character:lisa_loud cover fanfiction:platz_eins frowning half-closed_eyes solo text title_card", + "width" : 1078, +}, + +) diff --git a/test/results/loungeunderwear.py b/test/results/loungeunderwear.py new file mode 100644 index 00000000..521de25c --- /dev/null +++ b/test/results/loungeunderwear.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://loungeunderwear.com/collections/apparel", + "#category": ("shopify", "loungeunderwear", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://de.loungeunderwear.com/products/ribbed-crop-top-black", + "#category": ("shopify", "loungeunderwear", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/luscious.py b/test/results/luscious.py new file mode 100644 index 00000000..d5429612 --- /dev/null +++ b/test/results/luscious.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import luscious +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://luscious.net/albums/okinami-no-koigokoro_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#pattern" : r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", + + "album" : { + "__typename" : "Album", + "audiences" : list, + "content" : "Hentai", + "cover" : r"re:https://\w+.luscious.net/.+/277031/", + "created" : 1479625853, + "created_by" : "NTRshouldbeillegal", + "date" : "dt:2016-11-20 07:10:53", + "description" : "Enjoy.", + "download_url" : r"re:/download/(r/)?824778/277031/", + "genres" : list, + "id" : 277031, + "is_manga" : True, + "labels" : list, + "language" : "English", + "like_status" : "none", + "modified" : int, + "permissions" : list, + "rating" : float, + "slug" : "okinami-no-koigokoro", + "status" : None, + "tags" : list, + "title" : "Okinami no Koigokoro", + "url" : "/albums/okinami-no-koigokoro_277031/", + "marked_for_deletion" : False, + "marked_for_processing" : False, + "number_of_animated_pictures": 0, + "number_of_favorites" : int, + "number_of_pictures" : 18, + }, + "aspect_ratio" : r"re:\d+:\d+", + "category" : "luscious", + "created" : int, + "date" : datetime.datetime, + "height" : int, + "id" : int, + "is_animated" : False, + "like_status" : "none", + "position" : int, + "resolution" : r"re:\d+x\d+", + "status" : None, + "tags" : list, + "thumbnail" : str, + "title" : str, + "width" : int, + "number_of_comments": int, + "number_of_favorites": int, +}, + +{ + "#url" : "https://luscious.net/albums/not-found_277035/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://members.luscious.net/albums/login-required_323871/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, + "#count" : 64, +}, + +{ + "#url" : "https://www.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", + "#category": ("", "luscious", "album"), + "#class" : luscious.LusciousAlbumExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/list/", + "#category": ("", "luscious", "search"), + "#class" : luscious.LusciousSearchExtractor, +}, + +{ + "#url" : "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", + "#category": ("", "luscious", "search"), + "#class" : luscious.LusciousSearchExtractor, + "#pattern" : luscious.LusciousAlbumExtractor.pattern, + "#range" : "41-60", + "#count" : 20, +}, + +) diff --git a/test/results/mangadex.py b/test/results/mangadex.py new file mode 100644 index 00000000..c6a9e53a --- /dev/null +++ b/test/results/mangadex.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangadex +from gallery_dl import exception +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", +}, + +{ + "#url" : "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", + "#comment" : "oneshot", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#count" : 64, + "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", +}, + +{ + "#url" : "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", + "#comment" : "MANGA Plus (#1154)", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", + "#comment" : "'externalUrl', but still downloadable / 404 (#2503)", + "#category": ("", "mangadex", "chapter"), + "#class" : mangadex.MangadexChapterExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment" : "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : ">= 5", + + "manga" : "Souten no Koumori", + "manga_id" : "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "title" : r"re:One[Ss]hot", + "volume" : 0, + "chapter" : 0, + "chapter_minor": "", + "chapter_id" : str, + "date" : datetime.datetime, + "lang" : str, + "language" : str, + "artist" : ["Arakawa Hiromu"], + "author" : ["Arakawa Hiromu"], + "status" : "completed", + "tags" : [ + "Oneshot", + "Historical", + "Action", + "Martial Arts", + "Drama", + "Tragedy", + ], +}, + +{ + "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment" : "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#options" : {"lang": "fr,it"}, + "#count" : 2, + + "manga" : "Souten no Koumori", + "lang" : r"re:fr|it", + "language": r"re:French|Italian", +}, + +{ + "#url" : "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#options" : {"lang": "en"}, + "#count" : ">= 100", +}, + +{ + "#url" : "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", + "#category": ("", "mangadex", "manga"), + "#class" : mangadex.MangadexMangaExtractor, + "#count" : ">= 20", +}, + +{ + "#url" : "https://mangadex.org/title/feed", + "#category": ("", "mangadex", "feed"), + "#class" : mangadex.MangadexFeedExtractor, +}, + +) diff --git a/test/results/mangafox.py b/test/results/mangafox.py new file mode 100644 index 00000000..dc0cc9a0 --- /dev/null +++ b/test/results/mangafox.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangafox +import datetime + + +__tests__ = ( +{ + "#url" : "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, + "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", + "#sha1_content" : "5c50c252dcf12ffecf68801f4db8a2167265f66c", +}, + +{ + "#url" : "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, +}, + +{ + "#url" : "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", + "#category": ("", "mangafox", "chapter"), + "#class" : mangafox.MangafoxChapterExtractor, +}, + +{ + "#url" : "https://fanfox.net/manga/kanojo_mo_kanojo", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, + "#pattern" : mangafox.MangafoxChapterExtractor.pattern, + "#count" : ">=60", + + "author" : "HIROYUKI", + "chapter" : int, + "chapter_minor" : r"re:^(\.\d+)?$", + "chapter_string": r"re:(v\d+/)?c\d+", + "date" : datetime.datetime, + "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", + "lang" : "en", + "language" : "English", + "manga" : "Kanojo mo Kanojo", + "tags" : [ + "Comedy", + "Romance", + "School Life", + "Shounen", + ], + "volume" : int, +}, + +{ + "#url" : "https://mangafox.me/manga/shangri_la_frontier", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, + "#pattern" : mangafox.MangafoxChapterExtractor.pattern, + "#count" : ">=45", +}, + +{ + "#url" : "https://m.fanfox.net/manga/sentai_daishikkaku", + "#category": ("", "mangafox", "manga"), + "#class" : mangafox.MangafoxMangaExtractor, +}, + +) diff --git a/test/results/mangahere.py b/test/results/mangahere.py new file mode 100644 index 00000000..db17a6b0 --- /dev/null +++ b/test/results/mangahere.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangahere + + +__tests__ = ( +{ + "#url" : "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, + "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", + "#sha1_content" : "708d475f06893b88549cbd30df1e3f9428f2c884", +}, + +{ + "#url" : "https://www.mangahere.cc/manga/beastars/c196/1.html", + "#comment" : "URLs without HTTP scheme (#1070)", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, + "#pattern" : "https://zjcdn.mangahere.org/.*", +}, + +{ + "#url" : "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, +}, + +{ + "#url" : "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class" : mangahere.MangahereChapterExtractor, +}, + +{ + "#url" : "https://www.mangahere.cc/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#count" : 71, + "#sha1_url" : "9c2e54ec42e9a87ad53096c328b33c90750af3e4", + "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", +}, + +{ + "#url" : "https://www.mangahere.cc/manga/hiyokoi/#50", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#sha1_url" : "654850570aa03825cd57e2ae2904af489602c523", + "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", +}, + +{ + "#url" : "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", + "#comment" : "adult filter (#556)", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, + "#pattern" : mangahere.MangahereChapterExtractor.pattern, + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, +}, + +{ + "#url" : "https://m.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class" : mangahere.MangahereMangaExtractor, +}, + +) diff --git a/test/results/mangakakalot.py b/test/results/mangakakalot.py new file mode 100644 index 00000000..b0b8bada --- /dev/null +++ b/test/results/mangakakalot.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangakakalot + + +__tests__ = ( +{ + "#url" : "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", + "#category": ("", "mangakakalot", "chapter"), + "#class" : mangakakalot.MangakakalotChapterExtractor, + "#pattern" : r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", + "#count" : 9, + "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", +}, + +{ + "#url" : "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", + "#category": ("", "mangakakalot", "chapter"), + "#class" : mangakakalot.MangakakalotChapterExtractor, +}, + +{ + "#url" : "https://ww3.mangakakalot.tv/manga/manga-jk986845", + "#category": ("", "mangakakalot", "manga"), + "#class" : mangakakalot.MangakakalotMangaExtractor, + "#pattern" : mangakakalot.MangakakalotChapterExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://mangakakalot.tv/manga/lk921810", + "#category": ("", "mangakakalot", "manga"), + "#class" : mangakakalot.MangakakalotMangaExtractor, +}, + +) diff --git a/test/results/mangalife.py b/test/results/mangalife.py new file mode 100644 index 00000000..80226afe --- /dev/null +++ b/test/results/mangalife.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangasee +import datetime + + +__tests__ = ( +{ + "#url" : "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", + "#category": ("", "mangalife", "chapter"), + "#class" : mangasee.MangaseeChapterExtractor, + "#pattern" : r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", + "#count" : 13, + + "author" : ["ODA Eiichiro"], + "chapter" : 1063, + "chapter_minor" : "", + "chapter_string": "110630", + "count" : 13, + "date" : "dt:2022-10-16 17:32:54", + "extension" : "png", + "filename" : r"re:1063-0\d\d", + "genre" : [ + "Action", + "Adventure", + "Comedy", + "Drama", + "Fantasy", + "Shounen", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "One Piece", + "page" : int, + "title" : "", +}, + +{ + "#url" : "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "#category": ("", "mangalife", "manga"), + "#class" : mangasee.MangaseeMangaExtractor, + "#pattern" : mangasee.MangaseeChapterExtractor.pattern, + "#count" : ">= 50", + + "author" : ["Canno"], + "chapter" : int, + "chapter_minor" : r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date" : datetime.datetime, + "genre" : [ + "Comedy", + "Romance", + "School Life", + "Seinen", + "Shoujo Ai", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "title" : "", +}, + +) diff --git a/test/results/manganelo.py b/test/results/manganelo.py new file mode 100644 index 00000000..b5772656 --- /dev/null +++ b/test/results/manganelo.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import manganelo + + +__tests__ = ( +{ + "#url" : "https://chapmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", + "#count" : 25, + "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", +}, + +{ + "#url" : "https://chapmanganelo.com/manga-ti107776/chapter-4", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", + "#count" : 45, + "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", +}, + +{ + "#url" : "https://chapmanganato.com/manga-no991297/chapter-8", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, + "#count" : 20, + + "chapter" : 8, + "chapter_minor": "-1", +}, + +{ + "#url" : "https://readmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://manganelo.com/chapter/gamers/chapter_15", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://manganelo.com/chapter/gq921227/chapter_23", + "#category": ("", "manganelo", "chapter"), + "#class" : manganelo.ManganeloChapterExtractor, +}, + +{ + "#url" : "https://chapmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, + "#pattern" : manganelo.ManganeloChapterExtractor.pattern, + "#count" : ">= 25", +}, + +{ + "#url" : "https://m.manganelo.com/manga-ti107776", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, + "#pattern" : manganelo.ManganeloChapterExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://readmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +{ + "#url" : "https://manganelo.com/manga/read_otome_no_teikoku", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +{ + "#url" : "https://manganelo.com/manga/ol921234/", + "#category": ("", "manganelo", "manga"), + "#class" : manganelo.ManganeloMangaExtractor, +}, + +) diff --git a/test/results/mangapark.py b/test/results/mangapark.py new file mode 100644 index 00000000..432f535f --- /dev/null +++ b/test/results/mangapark.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangapark +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, + "#pattern" : r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", + "#count" : 70, + + "artist" : [], + "author" : ["Amano Kozue"], + "chapter" : 60, + "chapter_id" : 6710214, + "chapter_minor": ".2", + "count" : 70, + "date" : "dt:2022-01-15 09:25:03", + "extension" : "jpeg", + "filename" : str, + "genre" : [ + "adventure", + "comedy", + "drama", + "sci_fi", + "shounen", + "slice_of_life", + ], + "lang" : "en", + "language" : "English", + "manga" : "Aria", + "manga_id" : 114972, + "page" : int, + "source" : "Koala", + "title" : "Special Navigation - Aquaria Ii", + "volume" : 12, +}, + +{ + "#url" : "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class" : mangapark.MangaparkChapterExtractor, +}, + +{ + "#url" : "https://mangapark.net/title/114972-aria", + "#comment" : "'source' option", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, + "#pattern" : mangapark.MangaparkChapterExtractor.pattern, + "#count" : 141, + + "chapter" : int, + "chapter_id" : int, + "chapter_minor": str, + "date" : datetime.datetime, + "lang" : "en", + "language" : "English", + "manga_id" : 114972, + "source" : r"re:Horse|Koala", + "source_id" : int, + "title" : str, + "volume" : int, +}, + +{ + "#url" : "https://mangapark.net/title/114972-aria", + "#comment" : "'source' option", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, + "#options" : {"source": "koala"}, + "#pattern" : mangapark.MangaparkChapterExtractor.pattern, + "#count" : 70, + + "source" : "Koala", + "source_id": 15150116, +}, + +{ + "#url" : "https://mangapark.com/title/114972-", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.com/title/114972", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.com/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.org/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.io/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +{ + "#url" : "https://mangapark.me/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class" : mangapark.MangaparkMangaExtractor, +}, + +) diff --git a/test/results/mangaread.py b/test/results/mangaread.py new file mode 100644 index 00000000..40fe4e21 --- /dev/null +++ b/test/results/mangaread.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangaread +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 11, + + "manga" : "One Piece", + "title" : "", + "chapter" : 1053, + "chapter_minor": ".3", + "tags" : ["Oda Eiichiro"], + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 9, + + "manga" : "Kanan-sama wa Akumade Choroi", + "title" : "", + "chapter" : 10, + "chapter_minor": "", + "tags" : list, + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", + "#comment" : "^^ no whitespace", + "#category": ("", "mangaread", "chapter"), + "#class" : mangaread.MangareadChapterExtractor, + "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 6, + + "manga" : "Above All Gods", + "title" : "", + "chapter" : 146, + "chapter_minor": ".5", + "tags" : list, + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+(-.+)?/", + "#count" : ">= 13", + + "manga" : "Kanan-sama wa Akumade Choroi", + "author" : ["nonco"], + "artist" : ["nonco"], + "type" : "Manga", + "genres" : [ + "Comedy", + "Romance", + "Shounen", + "Supernatural", + ], + "rating" : float, + "release" : 2022, + "status" : "OnGoing", + "lang" : "en", + "language" : "English", + "manga_alt" : list, + "description": str, +}, + +{ + "#url" : "https://www.mangaread.org/manga/one-piece", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", + "#count" : ">= 1066", + + "manga" : "One Piece", + "author" : ["Oda Eiichiro"], + "artist" : ["Oda Eiichiro"], + "type" : "Manga", + "genres" : list, + "rating" : float, + "release" : 1997, + "status" : "OnGoing", + "lang" : "en", + "language" : "English", + "manga_alt" : ["One Piece"], + "description": str, +}, + +{ + "#url" : "https://www.mangaread.org/manga/doesnotexist", + "#category": ("", "mangaread", "manga"), + "#class" : mangaread.MangareadMangaExtractor, + "#exception": exception.HttpError, +}, + +) diff --git a/test/results/mangasee.py b/test/results/mangasee.py new file mode 100644 index 00000000..8cf51d4e --- /dev/null +++ b/test/results/mangasee.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangasee +import datetime + + +__tests__ = ( +{ + "#url" : "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", + "#category": ("", "mangasee", "chapter"), + "#class" : mangasee.MangaseeChapterExtractor, + "#pattern" : r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", + "#count" : 8, + + "author" : ["NARUMI Naru"], + "chapter" : 4, + "chapter_minor" : ".5", + "chapter_string": "100045", + "count" : 8, + "date" : "dt:2020-01-20 21:52:53", + "extension" : "png", + "filename" : r"re:0004\.5-00\d", + "genre" : [ + "Comedy", + "Fantasy", + "Harem", + "Romance", + "Shounen", + "Supernatural", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Tokyo Innocent", + "page" : int, + "title" : "", +}, + +{ + "#url" : "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "#category": ("", "mangasee", "manga"), + "#class" : mangasee.MangaseeMangaExtractor, + "#pattern" : mangasee.MangaseeChapterExtractor.pattern, + "#count" : ">= 17", + + "author" : ["TAKASE Masaya"], + "chapter" : int, + "chapter_minor" : r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date" : datetime.datetime, + "genre" : [ + "Comedy", + "Romance", + "School Life", + "Shounen", + "Slice of Life", + ], + "index" : "1", + "lang" : "en", + "language" : "English", + "manga" : "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "title" : "", +}, + +) diff --git a/test/results/mangoxo.py b/test/results/mangoxo.py new file mode 100644 index 00000000..b7c392a4 --- /dev/null +++ b/test/results/mangoxo.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mangoxo + + +__tests__ = ( +{ + "#url" : "https://www.mangoxo.com/album/lzVOv1Q9", + "#category": ("", "mangoxo", "album"), + "#class" : mangoxo.MangoxoAlbumExtractor, + "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", + + "channel": { + "id" : "gaxO16d8", + "name" : "Phoenix", + "cover": str, + }, + "album" : { + "id" : "lzVOv1Q9", + "name" : r"re:池永康晟 Ikenaga Yasunari 透出古朴", + "date" : "dt:2019-03-22 14:42:00", + "description": str, + }, + "id" : int, + "num" : int, + "count" : 65, +}, + +{ + "#url" : "https://www.mangoxo.com/phoenix/album", + "#category": ("", "mangoxo", "channel"), + "#class" : mangoxo.MangoxoChannelExtractor, + "#pattern" : mangoxo.MangoxoAlbumExtractor.pattern, + "#range" : "1-30", + "#count" : "> 20", +}, + +) diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py new file mode 100644 index 00000000..2ad0dc20 --- /dev/null +++ b/test/results/mastodonsocial.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://mastodon.social/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#pattern" : r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range" : "1-60", + "#count" : 60, +}, + +{ + "#url" : "https://mastodon.social/@yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#pattern" : r"https://mastodon\.social/media_proxy/\d+/original", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://mastodon.social/@id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/web/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class" : mastodon.MastodonUserExtractor, +}, + +{ + "#url" : "https://mastodon.social/bookmarks", + "#category": ("mastodon", "mastodon.social", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, + "#count" : ">= 20", + "#extractor": False, +}, + +{ + "#url" : "https://mastodon.social/@0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://mastodon.social/users/id:10843/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://mastodon.social/@jk/103794036899778366", + "#category": ("mastodon", "mastodon.social", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#count" : 4, + + "count": 4, + "num" : int, +}, + +) diff --git a/test/results/mememuseum.py b/test/results/mememuseum.py new file mode 100644 index 00000000..6a93d3f2 --- /dev/null +++ b/test/results/mememuseum.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://meme.museum/post/list/animated/1", + "#category": ("shimmie2", "mememuseum", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://meme\.museum/_images/\w+/\d+%20-%20", + "#count" : ">= 30", +}, + +{ + "#url" : "https://meme.museum/post/view/10243", + "#category": ("shimmie2", "mememuseum", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://meme\.museum/_images/105febebcd5ca791ee332adc49971f78/10243%20-%20g%20beard%20open_source%20richard_stallman%20stallman%20tagme%20text\.jpg", + "#sha1_content": "45565f3f141fc960a8ae1168b80e718a494c52d2", + + "extension" : "jpg", + "file_url" : "https://meme.museum/_images/105febebcd5ca791ee332adc49971f78/10243%20-%20g%20beard%20open_source%20richard_stallman%20stallman%20tagme%20text.jpg", + "filename" : "10243 - g beard open_source richard_stallman stallman tagme text", + "height" : 451, + "id" : 10243, + "md5" : "105febebcd5ca791ee332adc49971f78", + "size" : 0, + "subcategory": "post", + "tags" : "/g/ beard open_source richard_stallman stallman tagme text", + "width" : 480, +}, + +) diff --git a/test/results/michaelscameras.py b/test/results/michaelscameras.py new file mode 100644 index 00000000..42c76bbc --- /dev/null +++ b/test/results/michaelscameras.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://michaels.com.au/collections/microphones", + "#category": ("shopify", "michaelscameras", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", + "#category": ("shopify", "michaelscameras", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/misskeyio.py b/test/results/misskeyio.py new file mode 100644 index 00000000..3a28ae87 --- /dev/null +++ b/test/results/misskeyio.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://misskey.io/@lithla", + "#category": ("misskey", "misskey.io", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#pattern" : r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://misskey.io/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.io", "user"), + "#class" : misskey.MisskeyUserExtractor, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://misskey.io/@blooddj@pawoo.net/following", + "#category": ("misskey", "misskey.io", "following"), + "#class" : misskey.MisskeyFollowingExtractor, + "#count" : ">= 6", + "#extractor": False, +}, + +{ + "#url" : "https://misskey.io/notes/9bhqfo835v", + "#category": ("misskey", "misskey.io", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#pattern" : r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", + "#count" : 4, +}, + +{ + "#url" : "https://misskey.io/notes/9brq7z1re6", + "#category": ("misskey", "misskey.io", "note"), + "#class" : misskey.MisskeyNoteExtractor, +}, + +{ + "#url" : "https://misskey.io/my/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +{ + "#url" : "https://misskey.io/api/i/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/modcloth.py b/test/results/modcloth.py new file mode 100644 index 00000000..67ad082e --- /dev/null +++ b/test/results/modcloth.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://modcloth.com/collections/shoes", + "#category": ("shopify", "modcloth", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://modcloth.com/collections/shoes/products/heidii-brn", + "#category": ("shopify", "modcloth", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/myhentaigallery.py b/test/results/myhentaigallery.py new file mode 100644 index 00000000..6ddc2c2f --- /dev/null +++ b/test/results/myhentaigallery.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import myhentaigallery + + +__tests__ = ( +{ + "#url" : "https://myhentaigallery.com/gallery/thumbnails/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, + "#pattern" : r"https://images.myhentaicomics\.com/imagesgallery/images/[^/]+/original/\d+\.jpg", + + "artist" : list, + "count" : 11, + "gallery_id": 16247, + "group" : list, + "parodies" : list, + "tags" : ["Giantess"], + "title" : "Attack Of The 50ft Woman 1", +}, + +{ + "#url" : "https://myhentaigallery.com/gallery/show/16247/1", + "#category": ("", "myhentaigallery", "gallery"), + "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, +}, + +) diff --git a/test/results/myportfolio.py b/test/results/myportfolio.py new file mode 100644 index 00000000..71851293 --- /dev/null +++ b/test/results/myportfolio.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import myportfolio +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#sha1_url" : "acea0690c76db0e5cf267648cefd86e921bc3499", + "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", +}, + +{ + "#url" : "https://andrewling.myportfolio.com/", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#pattern" : r"https://andrewling\.myportfolio\.com/[^/?#+]+$", + "#count" : ">= 6", +}, + +{ + "#url" : "https://stevenilousphotography.myportfolio.com/society", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", + "#comment" : "custom domain", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#count" : 3, +}, + +{ + "#url" : "myportfolio:https://tooco.com.ar/", + "#category": ("", "myportfolio", "gallery"), + "#class" : myportfolio.MyportfolioGalleryExtractor, + "#pattern" : myportfolio.MyportfolioGalleryExtractor.pattern, + "#count" : ">= 40", +}, + +) diff --git a/test/results/naver.py b/test/results/naver.py new file mode 100644 index 00000000..81d18efd --- /dev/null +++ b/test/results/naver.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import naver + + +__tests__ = ( +{ + "#url" : "https://blog.naver.com/rlfqjxm0/221430673006", + "#category": ("", "naver", "post"), + "#class" : naver.NaverPostExtractor, + "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", +}, + +{ + "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class" : naver.NaverPostExtractor, + "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", +}, + +{ + "#url" : "https://blog.naver.com/gukjung", + "#category": ("", "naver", "blog"), + "#class" : naver.NaverBlogExtractor, + "#pattern" : naver.NaverPostExtractor.pattern, + "#range" : "1-12", + "#count" : 12, +}, + +{ + "#url" : "https://blog.naver.com/PostList.nhn?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class" : naver.NaverBlogExtractor, + "#pattern" : naver.NaverPostExtractor.pattern, + "#range" : "1-12", + "#count" : 12, +}, + +) diff --git a/test/results/naverwebtoon.py b/test/results/naverwebtoon.py new file mode 100644 index 00000000..889e5bce --- /dev/null +++ b/test/results/naverwebtoon.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import naverwebtoon + + +__tests__ = ( +{ + "#url" : "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count" : 14, + "#sha1_url" : "47a956ba8c7a837213d5985f50c569fcff986f75", + "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", + + "author" : ["김규삼"], + "artist" : ["김규삼"], + "comic" : "N의등대-눈의등대", + "count" : 14, + "episode" : "1", + "extension": "jpg", + "num" : int, + "tags" : [ + "스릴러", + "완결무료", + "완결스릴러", + ], + "title" : "n의 등대 - 눈의 등대 1화", + "title_id" : "26458", +}, + +{ + "#url" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", + "#count" : 1, + + "author" : ["kemi****"], + "artist" : [], + "comic" : "우니 모두의 이야기", + "count" : 1, + "episode" : "1", + "extension": "jpeg", + "filename" : "upload_7149856273586337846", + "num" : 1, + "tags" : [ + "일상툰", + "우니모두의이야기", + "퇴사", + "입사", + "신입사원", + "사회초년생", + "회사원", + "20대", + ], + "title" : "퇴사하다", + "title_id" : "765124", +}, + +{ + "#url" : "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", + "#category": ("", "naverwebtoon", "episode"), + "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", + "#count" : 1, +}, + +{ + "#url" : "https://comic.naver.com/webtoon/list?titleId=22073", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : 32, +}, + +{ + "#url" : "https://comic.naver.com/challenge/list?titleId=765124", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : 25, +}, + +{ + "#url" : "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", + "#category": ("", "naverwebtoon", "comic"), + "#class" : naverwebtoon.NaverwebtoonComicExtractor, + "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count" : ">= 12", +}, + +) diff --git a/test/results/newgrounds.py b/test/results/newgrounds.py new file mode 100644 index 00000000..3dac724d --- /dev/null +++ b/test/results/newgrounds.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import newgrounds + + +__tests__ = ( +{ + "#url" : "https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#sha1_url" : "57f182bcbbf2612690c3a54f16ffa1da5105245e", + "#sha1_content": "8f395e08333eb2457ba8d8b715238f8910221365", + + "artist" : ["tomfulp"], + "comment" : r"re:Consider this the bottom threshold for ", + "date" : "dt:2009-06-04 14:44:05", + "description": r"re:Consider this the bottom threshold for ", + "favorites" : int, + "filename" : "94_tomfulp_ryu-is-hawt", + "height" : 476, + "index" : 94, + "rating" : "e", + "score" : float, + "tags" : [ + "ryu", + "streetfighter", + ], + "title" : "Ryu is Hawt", + "type" : "article", + "user" : "tomfulp", + "width" : 447, +}, + +{ + "#url" : "https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#sha1_url": "57f182bcbbf2612690c3a54f16ffa1da5105245e", +}, + +{ + "#url" : "https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#count" : 2, + "#sha1_url": "84eec95e663041a80630df72719f231e157e5f5d", +}, + +{ + "#url" : "https://www.newgrounds.com/art/view/kekiiro/red", + "#comment" : "'adult' rated (#2456)", + "#category": ("", "newgrounds", "image"), + "#class" : newgrounds.NewgroundsImageExtractor, + "#options" : {"username": None}, + "#count" : 1, +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/595355", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/alternate/564000/564957_alternate_31\.mp4\?1359712249", + + "artist" : [ + "kickinthehead", + "danpaladin", + "tomfulp", + ], + "comment" : r"re:My fan trailer for Alien Hominid HD!", + "date" : "dt:2013-02-01 09:50:49", + "description": "Fan trailer for Alien Hominid HD!", + "favorites" : int, + "filename" : "564957_alternate_31", + "index" : 595355, + "rating" : "e", + "score" : float, + "tags" : [ + "alienhominid", + "trailer", + ], + "title" : "Alien Hominid Fan Trailer", + "type" : "movie", + "user" : "kickinthehead", +}, + +{ + "#url" : "https://www.newgrounds.com/audio/listen/609768", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#sha1_url": "f4c5490ae559a3b05e46821bb7ee834f93a43c95", + + "artist" : [ + "zj", + "tomfulp", + ], + "comment" : r"""re:RECORDED 12-09-2014 + +From The ZJ "Late """, + "date" : "dt:2015-02-23 19:31:59", + "description": "From The ZJ Report Show!", + "favorites" : int, + "index" : 609768, + "rating" : "", + "score" : float, + "tags" : [ + "fulp", + "interview", + "tom", + "zj", + ], + "title" : "ZJ Interviews Tom Fulp!", + "type" : "music.song", + "user" : "zj", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/161181/format/flash", + "#comment" : "flash animation (#1257)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/161000/161181_ddautta_mask__550x281_\.swf\?f1081628129", + + "type": "movie", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/758545", + "#comment" : "format selection (#1729)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"format": "720p"}, + "#pattern" : r"https://uploads\.ungrounded\.net/alternate/1482000/1482860_alternate_102516\.720p\.mp4\?\d+", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/717744", + "#comment" : "'adult' rated (#2456)", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"username": None}, + "#count" : 1, +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/829032", + "#comment" : "flash game", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#pattern" : r"https://uploads\.ungrounded\.net/829000/829032_picovsbeardx\.swf\?f1641968445", + "#range" : "1", + + "artist" : [ + "dungeonation", + "carpetbakery", + "animalspeakandrews", + "bill", + "chipollo", + "dylz49", + "gappyshamp", + "pinktophat", + "rad", + "shapeshiftingblob", + "tomfulp", + "voicesbycorey", + "psychogoldfish", + ], + "comment" : r"re:The children are expendable. Take out the ", + "date" : "dt:2022-01-10 23:00:57", + "description": "Bloodshed in The Big House that Blew...again!", + "favorites" : int, + "index" : 829032, + "post_url" : "https://www.newgrounds.com/portal/view/829032", + "rating" : "m", + "score" : float, + "tags" : [ + "assassin", + "boyfriend", + "darnell", + "nene", + "pico", + "picos-school", + ], + "title" : "PICO VS BEAR DX", + "type" : "game", + "url" : "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf?f1641968445", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/art", + "#category": ("", "newgrounds", "art"), + "#class" : newgrounds.NewgroundsArtExtractor, + "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, + "#count" : ">= 3", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/audio", + "#category": ("", "newgrounds", "audio"), + "#class" : newgrounds.NewgroundsAudioExtractor, + "#pattern" : r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3", + "#count" : ">= 4", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/movies", + "#category": ("", "newgrounds", "movies"), + "#class" : newgrounds.NewgroundsMoviesExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/games", + "#category": ("", "newgrounds", "games"), + "#class" : newgrounds.NewgroundsGamesExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com", + "#category": ("", "newgrounds", "user"), + "#class" : newgrounds.NewgroundsUserExtractor, + "#pattern" : "https://tomfulp.newgrounds.com/art$", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com", + "#category": ("", "newgrounds", "user"), + "#class" : newgrounds.NewgroundsUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : "https://tomfulp.newgrounds.com/(art|audio|movies)$", + "#count" : 3, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/art", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/audio", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/movies", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/", + "#category": ("", "newgrounds", "favorite"), + "#class" : newgrounds.NewgroundsFavoriteExtractor, +}, + +{ + "#url" : "https://tomfulp.newgrounds.com/favorites/following", + "#category": ("", "newgrounds", "following"), + "#class" : newgrounds.NewgroundsFollowingExtractor, + "#pattern" : newgrounds.NewgroundsUserExtractor.pattern, + "#range" : "76-125", + "#count" : 50, +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/art?terms=tree", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, + "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, + "#range" : "1-10", + "#count" : 10, + + "search_tags": "tree", +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/movies?terms=tree", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, + "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", + "#category": ("", "newgrounds", "search"), + "#class" : newgrounds.NewgroundsSearchExtractor, +}, + +) diff --git a/test/results/nhentai.py b/test/results/nhentai.py new file mode 100644 index 00000000..946acb3a --- /dev/null +++ b/test/results/nhentai.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nhentai + + +__tests__ = ( +{ + "#url" : "https://nhentai.net/g/147850/", + "#category": ("", "nhentai", "gallery"), + "#class" : nhentai.NhentaiGalleryExtractor, + "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", + + "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", + "title_en" : str, + "title_ja" : str, + "gallery_id": 147850, + "media_id" : 867789, + "count" : 16, + "date" : 1446050915, + "scanlator" : "", + "artist" : ["morris"], + "group" : list, + "parody" : list, + "characters": list, + "tags" : list, + "type" : "manga", + "lang" : "en", + "language" : "English", + "width" : int, + "height" : int, +}, + +{ + "#url" : "https://nhentai.net/tag/sole-female/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, + "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://nhentai.net/artist/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/group/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/parody/touhou-project/", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/character/patchouli-knowledge/popular", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/category/doujinshi/popular-today", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/language/english/popular-week", + "#category": ("", "nhentai", "tag"), + "#class" : nhentai.NhentaiTagExtractor, +}, + +{ + "#url" : "https://nhentai.net/search/?q=touhou", + "#category": ("", "nhentai", "search"), + "#class" : nhentai.NhentaiSearchExtractor, + "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://nhentai.net/favorites/", + "#category": ("", "nhentai", "favorite"), + "#class" : nhentai.NhentaiFavoriteExtractor, +}, + +) diff --git a/test/results/nijie.py b/test/results/nijie.py new file mode 100644 index 00000000..2b5785af --- /dev/null +++ b/test/results/nijie.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nijie +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://nijie.info/members.php?id=44", + "#category": ("Nijie", "nijie", "user"), + "#class" : nijie.NijieUserExtractor, +}, + +{ + "#url" : "https://nijie.info/members_illust.php?id=44", + "#category": ("Nijie", "nijie", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#sha1_url": "1553e5144df50a676f5947d02469299b401ad6c0", + + "artist_id" : 44, + "artist_name": "ED", + "date" : datetime.datetime, + "description": str, + "extension" : "jpg", + "filename" : str, + "image_id" : int, + "num" : int, + "tags" : list, + "title" : str, + "url" : r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", + "user_id" : 44, + "user_name" : "ED", +}, + +{ + "#url" : "https://nijie.info/members_illust.php?id=43", + "#category": ("Nijie", "nijie", "illustration"), + "#class" : nijie.NijieIllustrationExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://nijie.info/members_dojin.php?id=6782", + "#category": ("Nijie", "nijie", "doujin"), + "#class" : nijie.NijieDoujinExtractor, + "#count" : ">= 18", + + "user_id" : 6782, + "user_name": "ジョニー＠アビオン村", +}, + +{ + "#url" : "https://nijie.info/user_like_illust_view.php?id=44", + "#category": ("Nijie", "nijie", "favorite"), + "#class" : nijie.NijieFavoriteExtractor, + "#count" : ">= 16", + + "user_id" : 44, + "user_name": "ED", +}, + +{ + "#url" : "https://nijie.info/history_nuita.php?id=728995", + "#category": ("Nijie", "nijie", "nuita"), + "#class" : nijie.NijieNuitaExtractor, + "#range" : "1-10", + "#count" : 10, + + "user_id" : 728995, + "user_name": "莚", +}, + +{ + "#url" : "https://nijie.info/like_user_view.php", + "#category": ("Nijie", "nijie", "feed"), + "#class" : nijie.NijieFeedExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://nijie.info/like_my.php", + "#category": ("Nijie", "nijie", "followed"), + "#class" : nijie.NijieFollowedExtractor, +}, + +{ + "#url" : "https://nijie.info/view.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, + "#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743", + "#sha1_metadata": "41da5d0e178b04f01fe72460185df52fadc3c91b", + "#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", +}, + +{ + "#url" : "https://nijie.info/view.php?id=70724", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://nijie.info/view_popup.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class" : nijie.NijieImageExtractor, +}, + +) diff --git a/test/results/nitter1d4us.py b/test/results/nitter1d4us.py new file mode 100644 index 00000000..4c6c3d12 --- /dev/null +++ b/test/results/nitter1d4us.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.1d4.us/supernaturepics", + "#category": ("nitter", "nitter.1d4.us", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#range" : "1", + + "user": {"id": "2976459548"}, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/with_replies", + "#category": ("nitter", "nitter.1d4.us", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/media", + "#category": ("nitter", "nitter.1d4.us", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/supernaturepics/search", + "#category": ("nitter", "nitter.1d4.us", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.1d4.us/playpokemon/status/1263832915173048321", + "#comment" : "content with emoji, newlines, hashtags (#338)", + "#category": ("nitter", "nitter.1d4.us", "tweet"), + "#class" : nitter.NitterTweetExtractor, + + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! + +You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! + +❓🎁➡️ """, +}, + +{ + "#url" : "https://nitter.1d4.us/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("nitter", "nitter.1d4.us", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"https://nitter\.1d4\.us/pic/orig/enc/bWVkaWEvRWFL\w+LmpwZw==", + "#count" : 4, + + "filename": r"re:EaK.{12}", +}, + +) diff --git a/test/results/nitterit.py b/test/results/nitterit.py new file mode 100644 index 00000000..79d6937f --- /dev/null +++ b/test/results/nitterit.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.it/POTUS/status/1639409307878928384", + "#category": ("nitter", "nitter.it", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 0, +}, + +) diff --git a/test/results/nitterkavinrocks.py b/test/results/nitterkavinrocks.py new file mode 100644 index 00000000..22a72e2e --- /dev/null +++ b/test/results/nitterkavinrocks.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548", + "#category": ("nitter", "nitter.kavin.rocks", "tweets"), + "#class" : nitter.NitterTweetsExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/with_replies", + "#category": ("nitter", "nitter.kavin.rocks", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/media", + "#category": ("nitter", "nitter.kavin.rocks", "media"), + "#class" : nitter.NitterMediaExtractor, + "#pattern" : r"https://nitter\.kavin\.rocks/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.kavin.rocks/id:2976459548/search", + "#category": ("nitter", "nitter.kavin.rocks", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.kavin.rocks/ed1conf/status/1163841619336007680", + "#comment" : "Nitter tweet (#890)", + "#category": ("nitter", "nitter.kavin.rocks", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url" : "e115bd1c86c660064e392b05269bbcafcd8c8b7a", + "#sha1_content": "f29501e44d88437fe460f5c927b7543fda0f6e34", +}, + +) diff --git a/test/results/nitterlacontrevoiefr.py b/test/results/nitterlacontrevoiefr.py new file mode 100644 index 00000000..b2a81ec1 --- /dev/null +++ b/test/results/nitterlacontrevoiefr.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#pattern" : r"https://nitter\.lacontrevoie\.fr/pic/orig/media%2FCGMNYZvW0AIVoom\.jpg", + "#range" : "1", + "#sha1_url": "54f4b55f2099dcc248f3fb7bfacf1349e08d8e2d", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/with_replies", + "#category": ("nitter", "nitter.lacontrevoie.fr", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/media", + "#category": ("nitter", "nitter.lacontrevoie.fr", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/supernaturepics/search", + "#category": ("nitter", "nitter.lacontrevoie.fr", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/894001459754180609", + "#comment" : "4 images", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url": "9c51b3a4a1114535eb9b168bba97ad95db0d59ff", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1065692031626829824", + "#comment" : "video", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"ytdl:https://nitter\.lacontrevoie\.fr/video/[0-9A-F]{10,}/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1065691868439007232%2Fpu%2Fpl%2Fnv8hUQC1R0SjhzcZ.m3u8%3Ftag%3D5", + + "extension": "mp4", + "filename" : "nv8hUQC1R0SjhzcZ", +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1460044411165888515", + "#comment" : "deleted quote tweet (#2225)", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://nitter.lacontrevoie.fr/i/status/1486373748911575046", + "#comment" : "'Misleading' content", + "#category": ("nitter", "nitter.lacontrevoie.fr", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 4, +}, + +) diff --git a/test/results/nitternet.py b/test/results/nitternet.py new file mode 100644 index 00000000..2a1cccf6 --- /dev/null +++ b/test/results/nitternet.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter +import datetime + + +__tests__ = ( +{ + "#url" : "https://nitter.net/supernaturepics", + "#category": ("nitter", "nitter.net", "tweets"), + "#class" : nitter.NitterTweetsExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", + "#count" : 20, + + "author" : { + "name": "supernaturepics", + "nick": "Nature Pictures", + }, + "comments": int, + "content" : str, + "count" : 1, + "date" : datetime.datetime, + "likes" : int, + "quotes" : int, + "retweets": int, + "tweet_id": r"re:\d+", + "user" : { + "date" : "dt:2015-01-12 10:25:00", + "description" : "The very best nature pictures.", + "favourites_count": int, + "followers_count" : int, + "friends_count" : int, + "id" : "2976459548", + "name" : "supernaturepics", + "nick" : "Nature Pictures", + "profile_banner" : "https://nitter.net/pic/https%3A%2F%2Fpbs.twimg.com%2Fprofile_banners%2F2976459548%2F1421058583%2F1500x500", + "profile_image" : "https://nitter.net/pic/pbs.twimg.com%2Fprofile_images%2F554585280938659841%2FFLVAlX18.jpeg", + "statuses_count" : 1568, + "verified" : False, + }, +}, + +{ + "#url" : "https://nitter.net/supernaturepics/with_replies", + "#category": ("nitter", "nitter.net", "replies"), + "#class" : nitter.NitterRepliesExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/media", + "#category": ("nitter", "nitter.net", "media"), + "#class" : nitter.NitterMediaExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/search", + "#category": ("nitter", "nitter.net", "search"), + "#class" : nitter.NitterSearchExtractor, + "#pattern" : r"https://nitter\.net/pic/orig/media%2F[\w-]+\.(jpg|png)$", + "#range" : "1-20", +}, + +{ + "#url" : "https://nitter.net/supernaturepics/status/604341487988576256", + "#category": ("nitter", "nitter.net", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#sha1_url" : "3f2b64e175bf284aa672c3bb53ed275e470b919a", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", + + "comments" : 19, + "content" : "Big Wedeene River, Canada", + "count" : 1, + "date" : "dt:2015-05-29 17:40:00", + "extension": "jpg", + "filename" : "CGMNYZvW0AIVoom", + "likes" : int, + "num" : 1, + "quotes" : 10, + "retweets" : int, + "tweet_id" : "604341487988576256", + "url" : "https://nitter.net/pic/orig/media%2FCGMNYZvW0AIVoom.jpg", + "user" : { + "name": "supernaturepics", + "nick": "Nature Pictures", + }, +}, + +{ + "#url" : "https://nitter.net/StobiesGalaxy/status/1270755918330896395", + "#comment" : "'quoted' option (#854)", + "#category": ("nitter", "nitter.net", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#options" : {"quoted": True}, + "#pattern" : r"https://nitter\.net/pic/orig/media%2FEa[KG].+\.jpg", + "#count" : 8, +}, + +) diff --git a/test/results/nitterunixfoxeu.py b/test/results/nitterunixfoxeu.py new file mode 100644 index 00000000..d9b216e2 --- /dev/null +++ b/test/results/nitterunixfoxeu.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nitter + + +__tests__ = ( +{ + "#url" : "https://nitter.unixfox.eu/supernaturepics", + "#category": ("nitter", "nitter.unixfox.eu", "tweets"), + "#class" : nitter.NitterTweetsExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/with_replies", + "#category": ("nitter", "nitter.unixfox.eu", "replies"), + "#class" : nitter.NitterRepliesExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/media", + "#category": ("nitter", "nitter.unixfox.eu", "media"), + "#class" : nitter.NitterMediaExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/user/2976459548/search", + "#category": ("nitter", "nitter.unixfox.eu", "search"), + "#class" : nitter.NitterSearchExtractor, +}, + +{ + "#url" : "https://nitter.unixfox.eu/i/web/status/1170041925560258560", + "#comment" : "Reply to deleted tweet (#403, #838)", + "#category": ("nitter", "nitter.unixfox.eu", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#pattern" : r"https://nitter\.unixfox\.eu/pic/orig/media%2FEDzS7VrU0AAFL4_\.jpg", +}, + +{ + "#url" : "https://nitter.unixfox.eu/mightbecurse/status/1492954264909479936", + "#comment" : "age-restricted (#2354)", + "#category": ("nitter", "nitter.unixfox.eu", "tweet"), + "#class" : nitter.NitterTweetExtractor, + "#count" : 1, + + "date": "dt:2022-02-13 20:10:00", +}, + +) diff --git a/test/results/nozomi.py b/test/results/nozomi.py new file mode 100644 index 00000000..042c7920 --- /dev/null +++ b/test/results/nozomi.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nozomi + + +__tests__ = ( +{ + "#url" : "https://nozomi.la/post/3649262.html", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", + "#sha1_url" : "e5525e717aec712843be8b88592d6406ae9e60ba", + "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", + + "artist" : ["hammer (sunset beach)"], + "character": ["patchouli knowledge"], + "copyright": ["touhou"], + "dataid" : r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", + "date" : "dt:2016-07-26 02:32:03", + "extension": "webp", + "filename" : str, + "height" : 768, + "is_video" : False, + "postid" : 3649262, + "tags" : list, + "type" : "jpg", + "url" : str, + "width" : 1024, +}, + +{ + "#url" : "https://nozomi.la/post/25588032.html", + "#comment" : "multiple images per post", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#count" : 7, + "#sha1_url" : "fb956ccedcf2cf509739d26e2609e910244aa56c", + "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", +}, + +{ + "#url" : "https://nozomi.la/post/130309.html", + "#comment" : "empty 'date' (#1163)", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + + "date": None, +}, + +{ + "#url" : "https://nozomi.la/post/1647.html", + "#comment" : "gif", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", + "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", +}, + +{ + "#url" : "https://nozomi.la/post/2269847.html", + "#comment" : "video", + "#category": ("", "nozomi", "post"), + "#class" : nozomi.NozomiPostExtractor, + "#pattern" : r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", + "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", +}, + +{ + "#url" : "https://nozomi.la/", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/index-2.html", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/index-Popular-33.html", + "#category": ("", "nozomi", "index"), + "#class" : nozomi.NozomiIndexExtractor, +}, + +{ + "#url" : "https://nozomi.la/tag/3:1_aspect_ratio-1.html", + "#category": ("", "nozomi", "tag"), + "#class" : nozomi.NozomiTagExtractor, + "#pattern" : r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", + "#range" : "1-25", + "#count" : ">= 25", +}, + +{ + "#url" : "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", + "#category": ("", "nozomi", "search"), + "#class" : nozomi.NozomiSearchExtractor, + "#count" : ">= 5", +}, + +) diff --git a/test/results/nsfwalbum.py b/test/results/nsfwalbum.py new file mode 100644 index 00000000..e89b8fb4 --- /dev/null +++ b/test/results/nsfwalbum.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nsfwalbum + + +__tests__ = ( +{ + "#url" : "https://nsfwalbum.com/album/401611", + "#category": ("", "nsfwalbum", "album"), + "#class" : nsfwalbum.NsfwalbumAlbumExtractor, + "#range" : "1-5", + "#sha1_url" : "b0481fc7fad5982da397b6359fbed8421b8ba284", + "#sha1_metadata": "e98f9b0d473c00000831618d0235863b1dd78294", +}, + +) diff --git a/test/results/nudecollect.py b/test/results/nudecollect.py new file mode 100644 index 00000000..423c915f --- /dev/null +++ b/test/results/nudecollect.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import nudecollect + + +__tests__ = ( +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/image-4-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "image"), + "#class" : nudecollect.NudecollectImageExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00004-5896498214-43-9689595623/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/9879560327/nudecollect\.com\.jpg", + + "slug" : "20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust", + "title" : "20201220 Teenpornstorage Patritcy Vanessa Lesbian Lust", + "num" : 4, + "count" : 108, + "mirror": 43, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/image-10-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "image"), + "#class" : nudecollect.NudecollectImageExtractor, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px/index-mirror-67-125.html", + "#category": ("", "nudecollect", "album"), + "#class" : nudecollect.NudecollectAlbumExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00\d\d\d-5896498214-67-9689595623/20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px/9879560327/nudecollect\.com\.jpg", + "#count" : 125, + + "slug" : "20170219_TheWhiteBoxxx_Caprice_Tracy_Loves_Hot_ass_fingering_and_sensual_lesbian_sex_with_alluring_Czech_babes_x125_1080px", + "title" : "20170219 TheWhiteBoxxx Caprice Tracy Loves Hot ass fingering and sensual lesbian sex with alluring Czech babes x125 1080px", + "num" : int, + "mirror": 67, +}, + +{ + "#url" : "https://www.nudecollect.com/content/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/page-1-pics-108-mirror-43.html", + "#category": ("", "nudecollect", "album"), + "#class" : nudecollect.NudecollectAlbumExtractor, + "#pattern" : r"https://mirror\d+\.nudecollect\.com/showimage/nudecollect-8769086487/image00\d\d\d-5896498214-43-9689595623/20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust/9879560327/nudecollect\.com\.jpg", + "#count" : 108, + + "slug" : "20201220_Teenpornstorage_Patritcy_Vanessa_Lesbian_Lust", + "title" : "20201220 Teenpornstorage Patritcy Vanessa Lesbian Lust", + "num" : int, + "mirror": 43, +}, + +) diff --git a/test/results/ohpolly.py b/test/results/ohpolly.py new file mode 100644 index 00000000..3ce1968b --- /dev/null +++ b/test/results/ohpolly.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.ohpolly.com/collections/dresses-mini-dresses", + "#category": ("shopify", "ohpolly", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", + "#category": ("shopify", "ohpolly", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/omgmiamiswimwear.py b/test/results/omgmiamiswimwear.py new file mode 100644 index 00000000..27f4d54d --- /dev/null +++ b/test/results/omgmiamiswimwear.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.omgmiamiswimwear.com/collections/fajas", + "#category": ("shopify", "omgmiamiswimwear", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.omgmiamiswimwear.com/products/la-medusa-maxi-dress", + "#category": ("shopify", "omgmiamiswimwear", "product"), + "#class" : shopify.ShopifyProductExtractor, + "#pattern" : r"https://cdn\.shopify\.com/s/files/1/1819/6171/", + "#count" : 5, +}, + +) diff --git a/test/results/paheal.py b/test/results/paheal.py new file mode 100644 index 00000000..3ef0ec58 --- /dev/null +++ b/test/results/paheal.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import paheal + + +__tests__ = ( +{ + "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class" : paheal.PahealTagExtractor, + "#pattern" : r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20", + "#count" : ">= 15", +}, + +{ + "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class" : paheal.PahealTagExtractor, + "#options" : {"metadata": True}, + "#range" : "1", + + "date" : "dt:2018-01-07 07:04:05", + "duration" : 0.0, + "extension" : "jpg", + "filename" : "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "height" : 768, + "id" : 2446128, + "md5" : "b0ceda9d860df1d15b60293a7eb465c1", + "search_tags": "Ayane_Suzuki", + "size" : 205312, + "source" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", + "tags" : "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "uploader" : "XXXname", + "width" : 1024, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/481609", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + "#pattern" : r"https://tulip\.paheal\.net/_images/bbdc1c33410c2cdce7556c7990be26b7/481609%20-%20Azumanga_Daioh%20inanimate%20Osaka%20Vuvuzela\.jpg", + "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", + + "date" : "dt:2010-06-17 15:40:23", + "extension": "jpg", + "file_url" : r"re:https://tulip.paheal.net/_images/bbdc1c33410c", + "filename" : "481609 - Azumanga_Daioh inanimate Osaka Vuvuzela", + "height" : 660, + "id" : 481609, + "md5" : "bbdc1c33410c2cdce7556c7990be26b7", + "size" : 157389, + "source" : "", + "tags" : "Azumanga_Daioh inanimate Osaka Vuvuzela", + "uploader" : "CaptainButtface", + "width" : 614, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/488534", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + + "date" : "dt:2010-06-25 13:51:17", + "height" : 800, + "md5" : "b39edfe455a0381110c710d6ed2ef57d", + "size" : 758989, + "source" : "http://www.furaffinity.net/view/4057821/", + "tags" : "inanimate thelost-dragon Vuvuzela", + "uploader": "leacheate_soup", + "width" : 1200, +}, + +{ + "#url" : "https://rule34.paheal.net/post/view/3864982", + "#comment" : "video", + "#category": ("shimmie2", "paheal", "post"), + "#class" : paheal.PahealPostExtractor, + "#pattern" : r"https://[\w]+\.paheal\.net/_images/7629fc0ff77e32637dde5bf4f992b2cb/3864982%20-%20animated%20Metal_Gear%20Metal_Gear_Solid_V%20Quiet%20Vg_erotica%20webm\.webm", + + "date" : "dt:2020-09-06 01:59:03", + "duration" : 30.0, + "extension": "webm", + "height" : 2500, + "id" : 3864982, + "md5" : "7629fc0ff77e32637dde5bf4f992b2cb", + "size" : 18454938, + "source" : "https://twitter.com/VG_Worklog/status/1302407696294055936", + "tags" : "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", + "uploader" : "justausername", + "width" : 1768, +}, + +) diff --git a/test/results/palanq.py b/test/results/palanq.py new file mode 100644 index 00000000..046e6e88 --- /dev/null +++ b/test/results/palanq.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://archive.palanq.win/c/thread/4209598/", + "#category": ("foolfuuka", "palanq", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", +}, + +{ + "#url" : "https://archive.palanq.win/c/", + "#category": ("foolfuuka", "palanq", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://archive.palanq.win/_/search/text/test/", + "#category": ("foolfuuka", "palanq", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://archive.palanq.win/c/gallery", + "#category": ("foolfuuka", "palanq", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/patreon.py b/test/results/patreon.py new file mode 100644 index 00000000..adad12e3 --- /dev/null +++ b/test/results/patreon.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import patreon +import datetime +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.patreon.com/koveliana", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#range" : "1-25", + "#count" : ">= 25", + + "attachments" : list, + "comment_count": int, + "content" : str, + "creator" : dict, + "date" : datetime.datetime, + "id" : int, + "images" : list, + "like_count" : int, + "post_type" : str, + "published_at" : str, + "title" : str, +}, + +{ + "#url" : "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#count" : 1, + + "date": "dt:2020-03-30 21:21:44", +}, + +{ + "#url" : "https://www.patreon.com/kovelianot", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.patreon.com/user?u=2931440", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, +}, + +{ + "#url" : "https://www.patreon.com/user/posts/?u=2931440", + "#category": ("", "patreon", "creator"), + "#class" : patreon.PatreonCreatorExtractor, +}, + +{ + "#url" : "https://www.patreon.com/home", + "#category": ("", "patreon", "user"), + "#class" : patreon.PatreonUserExtractor, +}, + +{ + "#url" : "https://www.patreon.com/posts/precious-metal-23563293", + "#comment" : "postfile + attachments", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://www.patreon.com/posts/56127163", + "#comment" : "postfile + content", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#count" : 3, + + "filename": r"re:^(?!1).+$", +}, + +{ + "#url" : "https://www.patreon.com/posts/free-post-12497641", + "#comment" : "tags (#1539)", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + + "tags": ["AWMedia"], +}, + +{ + "#url" : "https://www.patreon.com/posts/not-found-123", + "#category": ("", "patreon", "post"), + "#class" : patreon.PatreonPostExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/pawoo.py b/test/results/pawoo.py new file mode 100644 index 00000000..5a9bfcaa --- /dev/null +++ b/test/results/pawoo.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import mastodon + + +__tests__ = ( +{ + "#url" : "https://pawoo.net/@yoru_nine/", + "#category": ("mastodon", "pawoo", "user"), + "#class" : mastodon.MastodonUserExtractor, + "#range" : "1-60", + "#count" : 60, +}, + +{ + "#url" : "https://pawoo.net/bookmarks", + "#category": ("mastodon", "pawoo", "bookmark"), + "#class" : mastodon.MastodonBookmarkExtractor, +}, + +{ + "#url" : "https://pawoo.net/users/yoru_nine/following", + "#category": ("mastodon", "pawoo", "following"), + "#class" : mastodon.MastodonFollowingExtractor, +}, + +{ + "#url" : "https://pawoo.net/@yoru_nine/105038878897832922", + "#category": ("mastodon", "pawoo", "status"), + "#class" : mastodon.MastodonStatusExtractor, + "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", +}, + +) diff --git a/test/results/photobucket.py b/test/results/photobucket.py new file mode 100644 index 00000000..90b5b83f --- /dev/null +++ b/test/results/photobucket.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import photobucket + + +__tests__ = ( +{ + "#url" : "https://s369.photobucket.com/user/CrpyLrkr/library", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#pattern" : r"https?://[oi]+\d+.photobucket.com/albums/oo139/", + "#count" : ">= 50", +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/library/", + "#comment" : "subalbums of main 'directory'", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#options" : {"image-filter": "False"}, + "#pattern" : photobucket.PhotobucketAlbumExtractor.pattern, + "#count" : 1, +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/library/Basketball", + "#comment" : "subalbums of subalbum without images", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#pattern" : photobucket.PhotobucketAlbumExtractor.pattern, + "#count" : ">= 9", +}, + +{ + "#url" : "https://s1277.photobucket.com/user/sinisterkat44/library/", + "#comment" : "private (missing JSON data)", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://s1110.photobucket.com/user/chndrmhn100/library/Chandu%20is%20the%20King?sort=3&page=1", + "#category": ("", "photobucket", "album"), + "#class" : photobucket.PhotobucketAlbumExtractor, +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/media/Untitled-3-1.jpg.html", + "#category": ("", "photobucket", "image"), + "#class" : photobucket.PhotobucketImageExtractor, + "#sha1_url" : "3b647deeaffc184cc48c89945f67574559c9051f", + "#sha1_metadata": "69732741b2b351db7ecaa77ace2fdb39f08ca5a3", +}, + +{ + "#url" : "https://s271.photobucket.com/user/lakerfanryan/media/IsotopeswBros.jpg.html?sort=3&o=2", + "#category": ("", "photobucket", "image"), + "#class" : photobucket.PhotobucketImageExtractor, + "#sha1_url" : "12c1890c09c9cdb8a88fba7eec13f324796a8d7b", + "#sha1_metadata": "61200a223df6c06f45ac3d30c88b3f5b048ce9a8", +}, + +) diff --git a/test/results/photovogue.py b/test/results/photovogue.py new file mode 100644 index 00000000..6898f5d9 --- /dev/null +++ b/test/results/photovogue.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import photovogue +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class" : photovogue.PhotovogueUserExtractor, +}, + +{ + "#url" : "https://vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class" : photovogue.PhotovogueUserExtractor, + "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", + + "date" : datetime.datetime, + "favorite_count" : int, + "favorited" : list, + "id" : int, + "image_id" : str, + "is_favorite" : False, + "orientation" : r"re:portrait|landscape", + "photographer" : { + "biography" : "Born in 1995. Live in Bologna.", + "city" : "Bologna", + "country_id" : 106, + "favoritedCount": int, + "id" : 221252, + "isGold" : bool, + "isPro" : bool, + "latitude" : str, + "longitude" : str, + "name" : "Arianna Mattarozzi", + "user_id" : "38cb0601-4a85-453c-b7dc-7650a037f2ab", + "websites" : list, + }, + "photographer_id": 221252, + "tags" : list, + "title" : str, +}, + +) diff --git a/test/results/picarto.py b/test/results/picarto.py new file mode 100644 index 00000000..07456260 --- /dev/null +++ b/test/results/picarto.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import picarto +import datetime + + +__tests__ = ( +{ + "#url" : "https://picarto.tv/fnook/gallery/default/", + "#category": ("", "picarto", "gallery"), + "#class" : picarto.PicartoGalleryExtractor, + "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", + "#count" : ">= 7", + + "date": datetime.datetime, +}, + +) diff --git a/test/results/piczel.py b/test/results/piczel.py new file mode 100644 index 00000000..84d1d01b --- /dev/null +++ b/test/results/piczel.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import piczel + + +__tests__ = ( +{ + "#url" : "https://piczel.tv/gallery/Bikupan", + "#category": ("", "piczel", "user"), + "#class" : piczel.PiczelUserExtractor, + "#range" : "1-100", + "#count" : ">= 100", +}, + +{ + "#url" : "https://piczel.tv/gallery/Lulena/1114", + "#category": ("", "piczel", "folder"), + "#class" : piczel.PiczelFolderExtractor, + "#count" : ">= 4", +}, + +{ + "#url" : "https://piczel.tv/gallery/image/7807", + "#category": ("", "piczel", "image"), + "#class" : piczel.PiczelImageExtractor, + "#pattern" : r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", + "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", + + "created_at" : "2018-07-22T05:13:58.000Z", + "date" : "dt:2018-07-22 05:13:58", + "description" : None, + "extension" : "png", + "favorites_count" : int, + "folder_id" : 1113, + "id" : 7807, + "is_flash" : False, + "is_video" : False, + "multi" : False, + "nsfw" : False, + "num" : 0, + "password_protected": False, + "tags" : [ + "fanart", + "commission", + "altair", + "recreators", + ], + "title" : "Altair", + "user" : dict, + "views" : int, +}, + +) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py new file mode 100644 index 00000000..a7c73beb --- /dev/null +++ b/test/results/pillowfort.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pillowfort +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.pillowfort.social/posts/27510", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#pattern" : r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", + "#count" : 4, + + "avatar_url" : str, + "col" : 0, + "commentable" : True, + "comments_count" : int, + "community_id" : None, + "content" : str, + "created_at" : str, + "date" : datetime.datetime, + "deleted" : None, + "deleted_at" : None, + "deleted_by_mod" : None, + "deleted_for_flag_id": None, + "embed_code" : None, + "id" : int, + "last_activity" : str, + "last_activity_elapsed": str, + "last_edited_at" : str, + "likes_count" : int, + "media_type" : "picture", + "nsfw" : False, + "num" : int, + "original_post_id": None, + "original_post_user_id": None, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id" : 27510, + "post_type" : "picture", + "privacy" : "public", + "reblog_copy_info": list, + "rebloggable" : True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count" : int, + "row" : int, + "small_image_url" : None, + "tags" : list, + "time_elapsed" : str, + "timestamp" : str, + "title" : "What is Pillowfort.social?", + "updated_at" : str, + "url" : r"re:https://img3.pillowfort.social/posts/.*\.png", + "user_id" : 5, + "username" : "Staff", +}, + +{ + "#url" : "https://www.pillowfort.social/posts/1557500", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#options" : { + "external": True, + "inline" : False, + }, + "#pattern" : r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", +}, + +{ + "#url" : "https://www.pillowfort.social/posts/1672518", + "#category": ("", "pillowfort", "post"), + "#class" : pillowfort.PillowfortPostExtractor, + "#options" : {"inline": True}, + "#count" : 3, +}, + +{ + "#url" : "https://www.pillowfort.social/Pome", + "#category": ("", "pillowfort", "user"), + "#class" : pillowfort.PillowfortUserExtractor, + "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", + "#range" : "1-15", + "#count" : 15, +}, + +) diff --git a/test/results/pinterest.py b/test/results/pinterest.py new file mode 100644 index 00000000..26671bff --- /dev/null +++ b/test/results/pinterest.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pinterest +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pinterest.com/pin/858146903966145189/", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#sha1_url" : "afb3c26719e3a530bb0e871c480882a801a4e8a5", + "#sha1_content": [ + "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", + "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", + ], +}, + +{ + "#url" : "https://www.pinterest.com/pin/422564377542934214/", + "#comment" : "video pin (#1189)", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#pattern" : r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", +}, + +{ + "#url" : "https://www.pinterest.com/pin/858146903966145188/", + "#category": ("", "pinterest", "pin"), + "#class" : pinterest.PinterestPinExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/test-/", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/", + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/stuff/", + "#comment" : "board with sections (#835)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#options" : {"sections": True}, + "#count" : 4, +}, + +{ + "#url" : "https://www.pinterest.de/g1952849/secret/", + "#comment" : "secret board (#1055)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/g1952848/test/", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, + "#exception": exception.GalleryDLException, +}, + +{ + "#url" : "https://www.pinterest.co.uk/hextra7519/based-animals/", + "#comment" : ".co.uk TLD (#914)", + "#category": ("", "pinterest", "board"), + "#class" : pinterest.PinterestBoardExtractor, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/", + "#category": ("", "pinterest", "user"), + "#class" : pinterest.PinterestUserExtractor, + "#pattern" : pinterest.PinterestBoardExtractor.pattern, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/_saved/", + "#category": ("", "pinterest", "user"), + "#class" : pinterest.PinterestUserExtractor, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/pins/", + "#category": ("", "pinterest", "allpins"), + "#class" : pinterest.PinterestAllpinsExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", + "#count" : 7, +}, + +{ + "#url" : "https://www.pinterest.de/digitalmomblog/_created/", + "#category": ("", "pinterest", "created"), + "#class" : pinterest.PinterestCreatedExtractor, + "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/stuff/section", + "#category": ("", "pinterest", "section"), + "#class" : pinterest.PinterestSectionExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://www.pinterest.com/search/pins/?q=nature", + "#category": ("", "pinterest", "search"), + "#class" : pinterest.PinterestSearchExtractor, + "#range" : "1-50", + "#count" : ">= 50", +}, + +{ + "#url" : "https://www.pinterest.com/pin/858146903966145189/#related", + "#category": ("", "pinterest", "related-pin"), + "#class" : pinterest.PinterestRelatedPinExtractor, + "#range" : "31-70", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://www.pinterest.com/g1952849/test-/#related", + "#category": ("", "pinterest", "related-board"), + "#class" : pinterest.PinterestRelatedBoardExtractor, + "#range" : "31-70", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://pin.it/Hvt8hgT", + "#category": ("", "pinterest", "pinit"), + "#class" : pinterest.PinterestPinitExtractor, + "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", +}, + +{ + "#url" : "https://pin.it/Hvt8hgS", + "#category": ("", "pinterest", "pinit"), + "#class" : pinterest.PinterestPinitExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/pinupgirlclothing.py b/test/results/pinupgirlclothing.py new file mode 100644 index 00000000..ac82ad53 --- /dev/null +++ b/test/results/pinupgirlclothing.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://pinupgirlclothing.com/collections/evening", + "#category": ("shopify", "pinupgirlclothing", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", + "#category": ("shopify", "pinupgirlclothing", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/pixhost.py b/test/results/pixhost.py new file mode 100644 index 00000000..82fbea98 --- /dev/null +++ b/test/results/pixhost.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://pixhost.to/show/190/130327671_test-.png", + "#category": ("imagehost", "pixhost", "image"), + "#class" : imagehosts.PixhostImageExtractor, + "#sha1_url" : "4e5470dcf6513944773044d40d883221bbc46cff", + "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", + "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", +}, + +{ + "#url" : "https://pixhost.to/gallery/jSMFq", + "#category": ("imagehost", "pixhost", "gallery"), + "#class" : imagehosts.PixhostGalleryExtractor, + "#pattern" : imagehosts.PixhostImageExtractor.pattern, + "#count" : 3, +}, + +) diff --git a/test/results/pixiv.py b/test/results/pixiv.py new file mode 100644 index 00000000..34c44f8c --- /dev/null +++ b/test/results/pixiv.py @@ -0,0 +1,520 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pixiv +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pixiv.net/en/users/173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/u/173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/member.php?id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/mypage.php#id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/#id=173530", + "#category": ("", "pixiv", "user"), + "#class" : pixiv.PixivUserExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/artworks", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", + "#comment" : "illusts with specific tag", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?id=173531", + "#comment" : "deleted account", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, + "#options" : {"metadata": True}, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/manga", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/illustrations", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class" : pixiv.PixivArtworksExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/avatar", + "#category": ("", "pixiv", "avatar"), + "#class" : pixiv.PixivAvatarExtractor, + "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/194921/background", + "#category": ("", "pixiv", "background"), + "#class" : pixiv.PixivBackgroundExtractor, + "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", +}, + +{ + "#url" : "https://pixiv.me/del_shannon", + "#category": ("", "pixiv", "me"), + "#class" : pixiv.PixivMeExtractor, + "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", +}, + +{ + "#url" : "https://pixiv.me/del_shanno", + "#category": ("", "pixiv", "me"), + "#class" : pixiv.PixivMeExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/artworks/966412", + "#comment" : "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#sha1_url" : "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", + + "date" : "dt:2008-06-12 15:29:13", + "date_url": "dt:2008-06-12 15:29:13", +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", + "#comment" : "ugoira", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#sha1_url": "7267695a985c4db8759bebcf8d21dbdd2d2317ef", + + "frames" : list, + "date" : "dt:2018-01-14 15:06:08", + "date_url": "dt:2018-01-15 04:24:48", +}, + +{ + "#url" : "https://www.pixiv.net/artworks/966412", + "#comment" : "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, + "#options" : {"related": True}, + "#range" : "1-10", + "#count" : ">= 10", +}, + +{ + "#url" : "https://www.pixiv.net/en/artworks/966412", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/i/966412", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://img.pixiv.net/img/soundcross/42626136.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", + "#category": ("", "pixiv", "work"), + "#class" : pixiv.PixivWorkExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/bookmarks/artworks", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=173530", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "85a3104eaaaf003c7b3947117ca2f1f0b1cfc949", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", + "#comment" : "bookmarks with specific tag", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", + "#comment" : "bookmarks with specific tag (legacy url)", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php", + "#comment" : "own bookmarks", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, + "#options" : {"metadata-bookmark": True}, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + + "tags_bookmark": [ + "47", + "hitman", + ], +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?tag=foobar", + "#comment" : "own bookmarks with tag (#596)", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/173530/following", + "#comment" : "followed users (#515)", + "#category": ("", "pixiv", "following"), + "#class" : pixiv.PixivFavoriteExtractor, + "#pattern" : pixiv.PixivUserExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://www.pixiv.net/bookmark.php?id=173530&type=user", + "#comment" : "followed users (legacy url) (#515)", + "#category": ("", "pixiv", "following"), + "#class" : pixiv.PixivFavoriteExtractor, + "#pattern" : pixiv.PixivUserExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark.php?id=173530", + "#comment" : "touch URLs", + "#category": ("", "pixiv", "favorite"), + "#class" : pixiv.PixivFavoriteExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark.php", + "#category": ("", "pixiv", "bookmark"), + "#class" : pixiv.PixivFavoriteExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/ranking.php?mode=unknown", + "#category": ("", "pixiv", "ranking"), + "#class" : pixiv.PixivRankingExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/en/tags/Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/search.php?word=Original", + "#category": ("", "pixiv", "search"), + "#class" : pixiv.PixivSearchExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class" : pixiv.PixivFollowExtractor, +}, + +{ + "#url" : "https://touch.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class" : pixiv.PixivFollowExtractor, +}, + +{ + "#url" : "https://www.pixivision.net/en/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class" : pixiv.PixivPixivisionExtractor, +}, + +{ + "#url" : "https://pixivision.net/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class" : pixiv.PixivPixivisionExtractor, + "#count" : 7, + + "pixivision_id" : "2791", + "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", +}, + +{ + "#url" : "https://www.pixiv.net/user/10509347/series/21859", + "#category": ("", "pixiv", "series"), + "#class" : pixiv.PixivSeriesExtractor, + "#range" : "1-10", + "#count" : 10, + + "num_series": int, + "series" : { + "canonical" : "https://www.pixiv.net/user/10509347/series/21859", + "description": str, + "ogp" : dict, + "title" : "先輩がうざい後輩の話", + "total" : int, + "twitter" : dict, + }, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=19612040", + "#comment" : "full series", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#count" : 1, + "#sha1_content": "8c818474153cbd2f221ee08766e1d634c821d8b4", + + "caption" : r"re:「無能な名無し」と呼ばれ虐げられて育った鈴\(すず\)は、", + "comment_access_control": 0, + "create_date" : "2023-04-02T15:18:58+09:00", + "date" : "dt:2023-04-02 06:18:58", + "id" : 19612040, + "is_bookmarked" : False, + "is_muted" : False, + "is_mypixiv_only" : False, + "is_original" : True, + "is_x_restricted" : False, + "novel_ai_type" : 1, + "page_count" : 1, + "rating" : "General", + "restrict" : 0, + "series" : { + "id" : 10278364, + "title": "龍の贄嫁〜無能な名無しと虐げられていましたが、どうやら異母妹に霊力を搾取されていたようです〜", + }, + "tags" : [ + "和風ファンタジー", + "溺愛", + "神様", + "ヤンデレ", + "執着", + "異能", + "ざまぁ", + "学園", + "神嫁", + ], + "text_length" : 5974, + "title" : "異母妹から「無能な名無し」と虐げられていた私、どうやら異母妹に霊力を搾取されていたようです（１）", + "user" : { + "account": "yukinaga_chifuyu", + "id" : 77055466, + }, + "visible" : True, + "x_restrict" : 0, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", + "#comment" : "embeds", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#options" : {"embeds": True}, + "#count" : 3, +}, + +{ + "#url" : "https://www.pixiv.net/novel/show.php?id=19612040", + "#comment" : "full series", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, + "#options" : {"full-series": True}, + "#count" : 4, +}, + +{ + "#url" : "https://www.pixiv.net/n/19612040", + "#comment" : "short URL", + "#category": ("", "pixiv", "novel"), + "#class" : pixiv.PixivNovelExtractor, +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/novels", + "#category": ("", "pixiv", "novel-user"), + "#class" : pixiv.PixivNovelUserExtractor, + "#pattern" : "^text:", + "#range" : "1-5", + "#count" : 5, +}, + +{ + "#url" : "https://www.pixiv.net/novel/series/10278364", + "#category": ("", "pixiv", "novel-series"), + "#class" : pixiv.PixivNovelSeriesExtractor, + "#count" : 4, + "#sha1_content": "b06abed001b3f6ccfb1579699e9a238b46d38ea2", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", + "#category": ("", "pixiv", "novel-bookmark"), + "#class" : pixiv.PixivNovelBookmarkExtractor, + "#count" : 1, + "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", +}, + +{ + "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", + "#category": ("", "pixiv", "novel-bookmark"), + "#class" : pixiv.PixivNovelBookmarkExtractor, +}, + +{ + "#url" : "https://sketch.pixiv.net/@nicoby", + "#category": ("", "pixiv", "sketch"), + "#class" : pixiv.PixivSketchExtractor, + "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", + "#count" : ">= 35", +}, + +) diff --git a/test/results/pixnet.py b/test/results/pixnet.py new file mode 100644 index 00000000..9c086526 --- /dev/null +++ b/test/results/pixnet.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pixnet + + +__tests__ = ( +{ + "#url" : "https://albertayu773.pixnet.net/album/photo/159443828", + "#category": ("", "pixnet", "image"), + "#class" : pixnet.PixnetImageExtractor, + "#sha1_url" : "156564c422138914c9fa5b42191677b45c414af4", + "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", + "#sha1_content" : "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/set/15078995", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#sha1_url" : "6535712801af47af51110542f4938a7cef44557f", + "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", +}, + +{ + "#url" : "https://anrine910070.pixnet.net/album/set/5917493", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#sha1_url" : "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", + "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", +}, + +{ + "#url" : "https://sky92100.pixnet.net/album/set/17492544", + "#comment" : "password-protected", + "#category": ("", "pixnet", "set"), + "#class" : pixnet.PixnetSetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/folder/1405768", + "#category": ("", "pixnet", "folder"), + "#class" : pixnet.PixnetFolderExtractor, + "#pattern" : pixnet.PixnetSetExtractor.pattern, + "#count" : ">= 15", +}, + +{ + "#url" : "https://albertayu773.pixnet.net/", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/blog", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, +}, + +{ + "#url" : "https://albertayu773.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, + "#pattern" : pixnet.PixnetFolderExtractor.pattern, + "#count" : ">= 30", +}, + +{ + "#url" : "https://anrine910070.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class" : pixnet.PixnetUserExtractor, + "#pattern" : pixnet.PixnetSetExtractor.pattern, + "#count" : ">= 14", +}, + +) diff --git a/test/results/plurk.py b/test/results/plurk.py new file mode 100644 index 00000000..11600bae --- /dev/null +++ b/test/results/plurk.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import plurk + + +__tests__ = ( +{ + "#url" : "https://www.plurk.com/plurkapi", + "#category": ("", "plurk", "timeline"), + "#class" : plurk.PlurkTimelineExtractor, + "#pattern" : "https?://.+", + "#count" : ">= 23", +}, + +{ + "#url" : "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class" : plurk.PlurkPostExtractor, + "#count" : 3, + "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", +}, + +{ + "#url" : "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class" : plurk.PlurkPostExtractor, + "#options" : {"comments": True}, + "#count" : ">= 210", +}, + +) diff --git a/test/results/poipiku.py b/test/results/poipiku.py new file mode 100644 index 00000000..08658bae --- /dev/null +++ b/test/results/poipiku.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import poipiku + + +__tests__ = ( +{ + "#url" : "https://poipiku.com/25049/", + "#category": ("", "poipiku", "user"), + "#class" : poipiku.PoipikuUserExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", + "#category": ("", "poipiku", "user"), + "#class" : poipiku.PoipikuUserExtractor, +}, + +{ + "#url" : "https://poipiku.com/25049/5864576.html", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", + + "count" : "1", + "description" : "", + "extension" : "png", + "filename" : "005864576_EWN1Y65gQ", + "num" : 1, + "post_category": "DOODLE", + "post_id" : "5864576", + "user_id" : "25049", + "user_name" : "ユキウサギ", +}, + +{ + "#url" : "https://poipiku.com/2166245/6411749.html", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", + "#count" : 4, + + "count" : "4", + "description" : "絵茶の産物ネタバレあるやつ", + "num" : int, + "post_category": "SPOILER", + "post_id" : "6411749", + "user_id" : "2166245", + "user_name" : "wadahito", +}, + +{ + "#url" : "https://poipiku.com/3572553/5776587.html", + "#comment" : "different warning button style", + "#category": ("", "poipiku", "post"), + "#class" : poipiku.PoipikuPostExtractor, + "#pattern" : r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", + "#count" : 3, + + "count" : "3", + "description" : "ORANGE OASISボスネタバレ
曲も大好き
2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", + "num" : int, + "post_category": "SPOILER", + "post_id" : "5776587", + "user_id" : "3572553", + "user_name" : "nagakun", +}, + +) diff --git a/test/results/ponybooru.py b/test/results/ponybooru.py new file mode 100644 index 00000000..6955bb49 --- /dev/null +++ b/test/results/ponybooru.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import philomena + + +__tests__ = ( +{ + "#url" : "https://ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, + "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", +}, + +{ + "#url" : "https://www.ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class" : philomena.PhilomenaPostExtractor, +}, + +{ + "#url" : "https://ponybooru.org/search?q=cute", + "#category": ("philomena", "ponybooru", "search"), + "#class" : philomena.PhilomenaSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://ponybooru.org/galleries/27", + "#category": ("philomena", "ponybooru", "gallery"), + "#class" : philomena.PhilomenaGalleryExtractor, + "#count" : ">= 24", +}, + +) diff --git a/test/results/pornhub.py b/test/results/pornhub.py new file mode 100644 index 00000000..e7aaf8da --- /dev/null +++ b/test/results/pornhub.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pornhub +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.pornhub.com/album/19289801", + "#category": ("", "pornhub", "gallery"), + "#class" : pornhub.PornhubGalleryExtractor, + "#pattern" : r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", + "#count" : ">= 300", + + "id" : int, + "num" : int, + "score" : int, + "views" : int, + "caption": str, + "user" : "Danika Mori", + "gallery": { + "id" : 19289801, + "score": int, + "views": int, + "tags" : list, + "title": "Danika Mori Best Moments", + }, +}, + +{ + "#url" : "https://www.pornhub.com/album/69040172", + "#category": ("", "pornhub", "gallery"), + "#class" : pornhub.PornhubGalleryExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://www.pornhub.com/gif/33643461", + "#category": ("", "pornhub", "gif"), + "#class" : pornhub.PornhubGifExtractor, + "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/033/643/461/33643461a\.webm", + + "date" : "dt:2020-10-31 00:00:00", + "extension": "webm", + "filename" : "33643461a", + "id" : "33643461", + "tags" : [ + "big boobs", + "lana rhoades", + ], + "title" : "Big boobs", + "url" : str, + "user" : "Lana Rhoades", +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori", + "#category": ("", "pornhub", "user"), + "#class" : pornhub.PornhubUserExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori/photos", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, + "#pattern" : pornhub.PornhubGalleryExtractor.pattern, + "#count" : ">= 6", +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/public", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/private", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/photos/favorites", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/model/bossgirl/photos", + "#category": ("", "pornhub", "photos"), + "#class" : pornhub.PornhubPhotosExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/pornstar/danika-mori/gifs", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, + "#pattern" : pornhub.PornhubGifExtractor.pattern, + "#count" : ">= 42", +}, + +{ + "#url" : "https://www.pornhub.com/users/flyings0l0/gifs", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, +}, + +{ + "#url" : "https://www.pornhub.com/model/bossgirl/gifs/video", + "#category": ("", "pornhub", "gifs"), + "#class" : pornhub.PornhubGifsExtractor, +}, + +) diff --git a/test/results/pornpics.py b/test/results/pornpics.py new file mode 100644 index 00000000..91a10ca0 --- /dev/null +++ b/test/results/pornpics.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pornpics + + +__tests__ = ( +{ + "#url" : "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", + "#category": ("", "pornpics", "gallery"), + "#class" : pornpics.PornpicsGalleryExtractor, + "#pattern" : r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", + + "categories": [ + "MILF", + "Amateur", + "Sexy", + "Outdoor", + ], + "channel" : "FTV MILFs", + "count" : 17, + "gallery_id": 62610699, + "models" : ["Danielle"], + "num" : int, + "slug" : "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + "tags" : [ + "Amateur MILF", + "Sexy MILF", + ], + "title" : "British beauty Danielle flashes hot breasts, ass and snatch in the forest", + "views" : int, +}, + +{ + "#url" : "https://pornpics.com/es/galleries/62610699", + "#category": ("", "pornpics", "gallery"), + "#class" : pornpics.PornpicsGalleryExtractor, + + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", +}, + +{ + "#url" : "https://www.pornpics.com/tags/summer-dress/", + "#category": ("", "pornpics", "tag"), + "#class" : pornpics.PornpicsTagExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://pornpics.com/fr/tags/summer-dress", + "#category": ("", "pornpics", "tag"), + "#class" : pornpics.PornpicsTagExtractor, +}, + +{ + "#url" : "https://www.pornpics.com/?q=nature", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.pornpics.com/channels/femjoy/", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.pornpics.com/pornstars/emma-brown/", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, + "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://pornpics.com/jp/?q=nature", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +{ + "#url" : "https://pornpics.com/it/channels/femjoy", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +{ + "#url" : "https://pornpics.com/pt/pornstars/emma-brown", + "#category": ("", "pornpics", "search"), + "#class" : pornpics.PornpicsSearchExtractor, +}, + +) diff --git a/test/results/pornreactor.py b/test/results/pornreactor.py new file mode 100644 index 00000000..0062d19a --- /dev/null +++ b/test/results/pornreactor.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://pornreactor.cc/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, + "#range" : "1-25", + "#count" : ">= 25", +}, + +{ + "#url" : "http://fapreactor.com/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/search?q=ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://fapreactor.com/search/ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class" : reactor.ReactorUserExtractor, + "#range" : "1-25", + "#count" : ">= 20", +}, + +{ + "#url" : "http://fapreactor.com/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://pornreactor.cc/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url" : "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", + "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", +}, + +{ + "#url" : "http://fapreactor.com/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", +}, + +) diff --git a/test/results/postimg.py b/test/results/postimg.py new file mode 100644 index 00000000..ba4b8a86 --- /dev/null +++ b/test/results/postimg.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://www.postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://pixxxels.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, +}, + +{ + "#url" : "https://postimg.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class" : imagehosts.PostimgImageExtractor, + "#sha1_url" : "72f3c8b1d6c6601a20ad58f35635494b4891a99e", + "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", + "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", +}, + +{ + "#url" : "https://postimg.cc/gallery/wxpDLgX", + "#category": ("imagehost", "postimg", "gallery"), + "#class" : imagehosts.PostimgGalleryExtractor, + "#pattern" : imagehosts.PostimgImageExtractor.pattern, + "#count" : 22, +}, + +) diff --git a/test/results/powermanga.py b/test/results/powermanga.py new file mode 100644 index 00000000..fb02b0cc --- /dev/null +++ b/test/results/powermanga.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolslide + + +__tests__ = ( +{ + "#url" : "https://read.powermanga.org/read/one_piece_digital_colour_comics/en/0/75/", + "#category": ("foolslide", "powermanga", "chapter"), + "#class" : foolslide.FoolslideChapterExtractor, + "#sha1_url" : "854c5817f8f767e1bccd05fa9d58ffb5a4b09384", + "#sha1_metadata": "a60c42f2634b7387899299d411ff494ed0ad6dbe", +}, + +{ + "#url" : "https://read.powermanga.org/series/one_piece_digital_colour_comics/", + "#category": ("foolslide", "powermanga", "manga"), + "#class" : foolslide.FoolslideMangaExtractor, + "#count" : ">= 1", + + "chapter" : int, + "chapter_minor" : str, + "chapter_string": str, + "group" : "PowerManga", + "lang" : "en", + "language" : "English", + "manga" : "One Piece Digital Colour Comics", + "title" : str, + "volume" : int, +}, + +) diff --git a/test/results/pururin.py b/test/results/pururin.py new file mode 100644 index 00000000..e57c7fe7 --- /dev/null +++ b/test/results/pururin.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import pururin + + +__tests__ = ( +{ + "#url" : "https://pururin.to/gallery/38661/iowant-2", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, + "#pattern" : r"https://i\.pururin\.to/38661/\d+\.jpg", + + "title" : r"re:I ?owant 2!!", + "title_en" : r"re:I ?owant 2!!", + "title_ja" : "", + "gallery_id": 38661, + "count" : 19, + "artist" : ["Shoda Norihiro"], + "group" : ["Obsidian Order"], + "parody" : ["Kantai Collection"], + "characters": [ + "Iowa", + "Teitoku", + ], + "tags" : list, + "type" : "Doujinshi", + "collection": ["I owant you!"], + "convention": ["C92"], + "rating" : float, + "uploader" : "demo", + "scanlator" : [ + "mrwayne", + "The Lost Light", + ], + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://pururin.to/gallery/7661/unisis-team-vanilla", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, + "#count" : 17, +}, + +{ + "#url" : "https://pururin.io/gallery/38661/iowant-2", + "#category": ("", "pururin", "gallery"), + "#class" : pururin.PururinGalleryExtractor, +}, + +) diff --git a/test/results/raidlondon.py b/test/results/raidlondon.py new file mode 100644 index 00000000..93aab265 --- /dev/null +++ b/test/results/raidlondon.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.raidlondon.com/collections/flats", + "#category": ("shopify", "raidlondon", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", + "#category": ("shopify", "raidlondon", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/rbt.py b/test/results/rbt.py new file mode 100644 index 00000000..5a81ad1a --- /dev/null +++ b/test/results/rbt.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://rbt.asia/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", +}, + +{ + "#url" : "https://archive.rebeccablacktech.com/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", +}, + +{ + "#url" : "https://rbt.asia/g/", + "#category": ("foolfuuka", "rbt", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://rbt.asia/_/search/text/test/", + "#category": ("foolfuuka", "rbt", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://rbt.asia/g/gallery/8", + "#category": ("foolfuuka", "rbt", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/reactor.py b/test/results/reactor.py new file mode 100644 index 00000000..7ff6dcce --- /dev/null +++ b/test/results/reactor.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reactor + + +__tests__ = ( +{ + "#url" : "http://reactor.cc/tag/gif", + "#category": ("reactor", "reactor", "tag"), + "#class" : reactor.ReactorTagExtractor, +}, + +{ + "#url" : "http://reactor.cc/search?q=Art", + "#category": ("reactor", "reactor", "search"), + "#class" : reactor.ReactorSearchExtractor, +}, + +{ + "#url" : "http://reactor.cc/user/Dioklet", + "#category": ("reactor", "reactor", "user"), + "#class" : reactor.ReactorUserExtractor, +}, + +{ + "#url" : "http://reactor.cc/post/4999736", + "#category": ("reactor", "reactor", "post"), + "#class" : reactor.ReactorPostExtractor, + "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", +}, + +) diff --git a/test/results/readcomiconline.py b/test/results/readcomiconline.py new file mode 100644 index 00000000..883ed321 --- /dev/null +++ b/test/results/readcomiconline.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import readcomiconline + + +__tests__ = ( +{ + "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", + "#category": ("", "readcomiconline", "issue"), + "#class" : readcomiconline.ReadcomiconlineIssueExtractor, + "#pattern" : r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", + "#count" : 36, + "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", +}, + +{ + "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h", + "#category": ("", "readcomiconline", "comic"), + "#class" : readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url" : "74eb8b9504b4084fcc9367b341300b2c52260918", + "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", +}, + +{ + "#url" : "https://readcomiconline.to/Comic/Bazooka-Jules", + "#category": ("", "readcomiconline", "comic"), + "#class" : readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url" : "2f66a467a772df4d4592e97a059ddbc3e8991799", + "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", +}, + +) diff --git a/test/results/realbooru.py b/test/results/realbooru.py new file mode 100644 index 00000000..e99634b4 --- /dev/null +++ b/test/results/realbooru.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://realbooru.com/index.php?page=post&s=list&tags=wine", + "#category": ("gelbooru_v02", "realbooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : ">= 64", +}, + +{ + "#url" : "https://realbooru.com/index.php?page=pool&s=show&id=1", + "#category": ("gelbooru_v02", "realbooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://realbooru.com/index.php?page=favorites&s=view&id=274", + "#category": ("gelbooru_v02", "realbooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://realbooru.com/index.php?page=post&s=view&id=668483", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : {"tags": True}, + "#pattern" : r"https://realbooru\.com//?images/dc/b5/dcb5c0ce9ec0bf74a6930608985f4719\.jpeg", + "#sha1_content": "7f5873ce3b6cd295ea2e81fcb49583098ea9c8da", + + "tags_general": "1girl blonde blonde_hair blue_eyes cute female female_only looking_at_viewer smile solo solo_female teeth", + "tags_model" : "jennifer_lawrence", +}, + +) diff --git a/test/results/recursive.py b/test/results/recursive.py new file mode 100644 index 00000000..2ad31dd4 --- /dev/null +++ b/test/results/recursive.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import recursive + + +__tests__ = ( +{ + "#url" : "recursive:https://pastebin.com/raw/FLwrCYsT", + "#category": ("", "recursive", ""), + "#class" : recursive.RecursiveExtractor, + "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", +}, + +) diff --git a/test/results/reddit.py b/test/results/reddit.py new file mode 100644 index 00000000..330ef5b6 --- /dev/null +++ b/test/results/reddit.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import reddit + + +__tests__ = ( +{ + "#url" : "https://www.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, + "#range" : "1-20", + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", + "#category": ("", "reddit", "subreddit-top"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://old.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://np.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://m.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class" : reddit.RedditSubredditExtractor, +}, + +{ + "#url" : "https://www.reddit.com/", + "#category": ("", "reddit", "home"), + "#class" : reddit.RedditHomeExtractor, + "#range" : "1-20", + "#count" : ">= 20", +}, + +{ + "#url" : "https://old.reddit.com/top/?sort=top&t=month", + "#category": ("", "reddit", "home-top"), + "#class" : reddit.RedditHomeExtractor, +}, + +{ + "#url" : "https://www.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.reddit.com/user/username/gilded/?sort=top&t=month", + "#category": ("", "reddit", "user-gilded"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://old.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://www.reddit.com/u/username/", + "#category": ("", "reddit", "user"), + "#class" : reddit.RedditUserExtractor, +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"comments": 500}, + "#pattern" : "https://", + "#count" : 3, +}, + +{ + "#url" : "https://www.reddit.com/gallery/hrrh23", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 3, + "#sha1_url" : "25b91ede15459470274dd17291424b037ed8b0ae", + "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "ytdl:https://v.redd.it/gyh95hiqc0b11", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"videos": "ytdl"}, + "#pattern" : "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment" : "video (dash)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#options" : {"videos": "dash"}, + "#pattern" : r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/gallery/icfgzv", + "#comment" : "deleted gallery (#953)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.reddit.com/r/araragi/comments/ib32hm", + "#comment" : "animated gallery items (#955)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"https://i\.redd\.it/\w+\.gif", + "#count" : 2, +}, + +{ + "#url" : "https://www.reddit.com/r/cosplay/comments/jvwaqr", + "#comment" : "'failed' gallery item (#1127)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/kpopfap/comments/qjj04q/", + "#comment" : "gallery with no 'media_metadata' (#2001)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", + "#comment" : "user page submission (#2301)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "https://i.redd.it/8fpgv17yqlh81.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.reddit.com/r/kittengifs/comments/12m0b8d", + "#comment" : "cross-posted video (#887, #3586, #3976)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : r"ytdl:https://v\.redd\.it/cvabpjacrvta1", +}, + +{ + "#url" : "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", + "#comment" : "preview.redd.it (#4470)", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, + "#pattern" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", +}, + +{ + "#url" : "https://old.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://np.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://m.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://redd.it/2a00np/", + "#category": ("", "reddit", "submission"), + "#class" : reddit.RedditSubmissionExtractor, +}, + +{ + "#url" : "https://i.redd.it/upjtjcx2npzz.jpg", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#sha1_url" : "0de614900feef103e580b632190458c0b62b641a", + "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", +}, + +{ + "#url" : "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#sha1_url" : "f24f25efcedaddeec802e46c60d77ef975dc52a5", + "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", +}, + +{ + "#url" : "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", + "#comment" : "preview.redd.it -> i.redd.it", + "#category": ("", "reddit", "image"), + "#class" : reddit.RedditImageExtractor, + "#pattern" : r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", +}, + +) diff --git a/test/results/redgifs.py b/test/results/redgifs.py new file mode 100644 index 00000000..2ae76485 --- /dev/null +++ b/test/results/redgifs.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import redgifs + + +__tests__ = ( +{ + "#url" : "https://www.redgifs.com/users/Natalifiction", + "#category": ("", "redgifs", "user"), + "#class" : redgifs.RedgifsUserExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#count" : ">= 100", +}, + +{ + "#url" : "https://v3.redgifs.com/users/lamsinka89", + "#category": ("", "redgifs", "user"), + "#class" : redgifs.RedgifsUserExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#count" : ">= 100", +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections/2631326bbd", + "#category": ("", "redgifs", "collection"), + "#class" : redgifs.RedgifsCollectionExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", + "#category": ("", "redgifs", "collection"), + "#class" : redgifs.RedgifsCollectionExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/users/boombah123/collections", + "#category": ("", "redgifs", "collections"), + "#class" : redgifs.RedgifsCollectionsExtractor, + "#pattern" : r"https://www\.redgifs\.com/users/boombah123/collections/\w+", + "#count" : ">= 3", +}, + +{ + "#url" : "https://www.redgifs.com/niches/boobs", + "#category": ("", "redgifs", "niches"), + "#class" : redgifs.RedgifsNichesExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/niches/ass", + "#category": ("", "redgifs", "niches"), + "#class" : redgifs.RedgifsNichesExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://www.redgifs.com/gifs/jav", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.redgifs.com/gifs/jav?order=best&verified=1", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://www.redgifs.com/browse?type=i&verified=y&order=top7", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://v3.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class" : redgifs.RedgifsSearchExtractor, +}, + +{ + "#url" : "https://redgifs.com/watch/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", + "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", +}, + +{ + "#url" : "https://www.redgifs.com/watch/desertedbaregraywolf", + "#comment" : "gallery (#4021)", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, + "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", + "#count" : 4, + + "num" : int, + "count" : 4, + "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", +}, + +{ + "#url" : "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +{ + "#url" : "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class" : redgifs.RedgifsImageExtractor, +}, + +) diff --git a/test/results/rule34.py b/test/results/rule34.py new file mode 100644 index 00000000..ca90e511 --- /dev/null +++ b/test/results/rule34.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", + "#category": ("gelbooru_v02", "rule34", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#pattern" : r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", + "#count" : 2, + "#sha1_content": [ + "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "622e80be3f496672c44aab5c47fbc6941c61bc79", + ], +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=pool&s=show&id=179", + "#category": ("gelbooru_v02", "rule34", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", + "#category": ("gelbooru_v02", "rule34", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://rule34.xxx/index.php?page=post&s=view&id=863", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : { + "tags" : True, + "notes": True, + }, + "#pattern" : r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", + "#sha1_content": [ + "a43f418aa350039af0d11cae501396a33bbe2201", + "67b516295950867e1c1ab6bc13b35d3b762ed2a3", + ], + + "tags_artist" : "reverse_noise yamu_(reverse_noise)", + "tags_character": "hong_meiling", + "tags_copyright": "touhou", + "tags_general" : str, + "tags_metadata" : "censored translated", + "notes" : [ + { + "body" : "It feels angry, I'm losing myself... It won't calm down!", + "height": 65, + "id" : 93586, + "width" : 116, + "x" : 22, + "y" : 333, + }, + { + "body" : "REPUTATION OF RAGE", + "height": 272, + "id" : 93587, + "width" : 199, + "x" : 78, + "y" : 442, + }, + ], +}, + +) diff --git a/test/results/rule34us.py b/test/results/rule34us.py new file mode 100644 index 00000000..43d8f66a --- /dev/null +++ b/test/results/rule34us.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import rule34us + + +__tests__ = ( +{ + "#url" : "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", + "#category": ("booru", "rule34us", "tag"), + "#class" : rule34us.Rule34usTagExtractor, + "#pattern" : r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", + "#count" : 10, +}, + +{ + "#url" : "https://rule34.us/index.php?r=posts/view&id=3709005", + "#category": ("booru", "rule34us", "post"), + "#class" : rule34us.Rule34usPostExtractor, + "#pattern" : r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", + "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", +}, + +{ + "#url" : "https://rule34.us/index.php?r=posts/view&id=4576310", + "#category": ("booru", "rule34us", "post"), + "#class" : rule34us.Rule34usPostExtractor, + "#pattern" : r"https://video\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", + + "extension" : "mp4", + "file_url" : str, + "filename" : "a294ff8e1f8e0efa041e5dc9d1480011", + "height" : "3982", + "id" : "4576310", + "md5" : "a294ff8e1f8e0efa041e5dc9d1480011", + "score" : r"re:\d+", + "tags" : "tagme, video", + "tags_general" : "video", + "tags_metadata": "tagme", + "uploader" : "Anonymous", + "width" : "3184", +}, + +) diff --git a/test/results/safebooru.py b/test/results/safebooru.py new file mode 100644 index 00000000..0c183204 --- /dev/null +++ b/test/results/safebooru.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url" : "17c61b386530cf4c30842c9f580d15ef1cd09586", + "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", +}, + +{ + "#url" : "https://safebooru.org/index.php?page=pool&s=show&id=11", + "#category": ("gelbooru_v02", "safebooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://safebooru.org/index.php?page=favorites&s=view&id=17567", + "#category": ("gelbooru_v02", "safebooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://safebooru.org/index.php?page=post&s=view&id=1169132", + "#category": ("gelbooru_v02", "safebooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#options" : {"tags": True}, + "#sha1_url" : "cf05e37a3c62b2d55788e2080b8eabedb00f999b", + "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", + + "tags_artist" : "kawanakajima", + "tags_character": "heath_ledger ronald_mcdonald the_joker", + "tags_copyright": "dc_comics mcdonald's the_dark_knight", + "tags_general" : str, +}, + +) diff --git a/test/results/sakugabooru.py b/test/results/sakugabooru.py new file mode 100644 index 00000000..73d38316 --- /dev/null +++ b/test/results/sakugabooru.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://www.sakugabooru.com/post/show/125570", + "#category": ("moebooru", "sakugabooru", "post"), + "#class" : moebooru.MoebooruPostExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/post?tags=nichijou", + "#category": ("moebooru", "sakugabooru", "tag"), + "#class" : moebooru.MoebooruTagExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/pool/show/54", + "#category": ("moebooru", "sakugabooru", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, +}, + +{ + "#url" : "https://www.sakugabooru.com/post/popular_recent", + "#category": ("moebooru", "sakugabooru", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/sankaku.py b/test/results/sankaku.py new file mode 100644 index 00000000..e4fbf5c1 --- /dev/null +++ b/test/results/sankaku.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sankaku +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://sankaku.app/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", + "#count" : 5, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://black.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://white.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://sankaku.app/ja?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://sankaku.app/no/?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", + "#comment" : "error on five or more tags", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#options" : {"username": None}, + "#exception": exception.StopExtraction, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", + "#comment" : "match arbitrary query parameters", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/?tags=date:2023-03-20", + "#comment" : "'date:' tags (#1790)", + "#category": ("booru", "sankaku", "tag"), + "#class" : sankaku.SankakuTagExtractor, + "#range" : "1", + "#count" : 1, +}, + +{ + "#url" : "https://sankaku.app/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, + "#count" : 5, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/pool/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class" : sankaku.SankakuPoolExtractor, +}, + +{ + "#url" : "https://sankaku.app/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + + "tags_artist" : ["bonocho"], + "tags_studio" : ["dc_comics"], + "tags_medium" : list, + "tags_copyright": list, + "tags_character": list, + "tags_general" : list, +}, + +{ + "#url" : "https://sankaku.app/post/show/21418978", + "#comment" : "'contentious_content'", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", +}, + +{ + "#url" : "https://sankaku.app/post/show/20758561", + "#comment" : "empty tags (#1617)", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#options" : {"tags": True}, + "#count" : 1, + + "tags" : list, + "tags_general": [ + "key(mangaka)", + "key(mangaka)", + ], +}, + +{ + "#url" : "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", + "#comment" : "md5 hexdigest instead of ID (#3952)", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, + "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count" : 1, + + "id" : 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", +}, + +{ + "#url" : "https://chan.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://chan.sankakucomplex.com/ja/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://white.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://black.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class" : sankaku.SankakuPostExtractor, +}, + +{ + "#url" : "https://sankaku.app/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class" : sankaku.SankakuBooksExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://beta.sankakucomplex.com/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class" : sankaku.SankakuBooksExtractor, +}, + +) diff --git a/test/results/sankakucomplex.py b/test/results/sankakucomplex.py new file mode 100644 index 00000000..f944df16 --- /dev/null +++ b/test/results/sankakucomplex.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sankakucomplex + + +__tests__ = ( +{ + "#url" : "https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#sha1_url" : "4a9ecc5ae917fbce469280da5b6a482510cae84d", + "#sha1_metadata": "bfe08310e7d9a572f568f6900e0ed0eb295aa2b3", +}, + +{ + "#url" : "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#sha1_url" : "a1e249173fd6c899a8134fcfbd9c925588a63f7c", + "#sha1_metadata": "e78fcc23c2711befc0969a45ea5082a29efccf68", +}, + +{ + "#url" : "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", + "#comment" : "videos (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#pattern" : r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", + "#range" : "26-", + "#count" : 5, +}, + +{ + "#url" : "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", + "#comment" : "youtube embeds (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class" : sankakucomplex.SankakucomplexArticleExtractor, + "#options" : {"embeds": True}, + "#pattern" : "https://www.youtube.com/embed/", + "#range" : "2-", + "#count" : 2, +}, + +{ + "#url" : "https://www.sankakucomplex.com/tag/cosplay/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, + "#pattern" : sankakucomplex.SankakucomplexArticleExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://www.sankakucomplex.com/category/anime/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, +}, + +{ + "#url" : "https://www.sankakucomplex.com/author/rift/page/5/", + "#category": ("", "sankakucomplex", "tag"), + "#class" : sankakucomplex.SankakucomplexTagExtractor, +}, + +) diff --git a/test/results/seiga.py b/test/results/seiga.py new file mode 100644 index 00000000..3749adc4 --- /dev/null +++ b/test/results/seiga.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import seiga +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, + "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", + "#count" : ">= 4", + + "user" : { + "id" : 39537793, + "message": str, + "name" : str, + }, + "clips" : int, + "comments" : int, + "count" : int, + "extension": None, + "image_id" : int, + "title" : str, + "views" : int, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/79433", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, +}, + +{ + "#url" : "https://sp.seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class" : seiga.SeigaUserExtractor, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", + "#sha1_content" : "d9202292012178374d57fb0126f6124387265297", +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im123", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://seiga.nicovideo.jp/seiga/im10877923", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, + "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", +}, + +{ + "#url" : "https://seiga.nicovideo.jp/image/source/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/thumb/5977527i", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +{ + "#url" : "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class" : seiga.SeigaImageExtractor, +}, + +) diff --git a/test/results/senmanga.py b/test/results/senmanga.py new file mode 100644 index 00000000..8192cfad --- /dev/null +++ b/test/results/senmanga.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import senmanga + + +__tests__ = ( +{ + "#url" : "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", + "#sha1_url" : "5f95140ff511d8497e2ec08fa7267c6bb231faec", + "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", + + "chapter" : "37A", + "count" : 2, + "extension": "", + "filename" : r"re:[12]", + "lang" : "ja", + "language" : "Japanese", + "manga" : "Bokura wa Minna Kawaisou", + "page" : int, +}, + +{ + "#url" : "http://raw.senmanga.com/Love-Lab/2016-03/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", + "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", + + "chapter" : "2016-03", + "count" : 9, + "extension": "", + "filename" : r"re:\d", + "manga" : "Renai Lab 恋愛ラボ", +}, + +{ + "#url" : "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", + + "chapter" : "1", + "count" : 65, + "extension": "jpg", + "filename" : r"re:\d+-\w+", + "manga" : "Akabane Honeko no Bodyguard", +}, + +{ + "#url" : "https://raw.senmanga.com/amama-cinderella/3", + "#comment" : "no http scheme ()", + "#category": ("", "senmanga", "chapter"), + "#class" : senmanga.SenmangaChapterExtractor, + "#pattern" : r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", + "#count" : 30, +}, + +) diff --git a/test/results/sexcom.py b/test/results/sexcom.py new file mode 100644 index 00000000..079bb5fa --- /dev/null +++ b/test/results/sexcom.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import sexcom + + +__tests__ = ( +{ + "#url" : "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", + "#comment" : "picture", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", + "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", + + "comments" : int, + "date" : "dt:2014-10-19 15:45:44", + "extension": "jpg", + "filename" : "7637609", + "likes" : int, + "pin_id" : 21241874, + "repins" : int, + "tags" : list, + "thumbnail": str, + "title" : "Sexy Ecchi Girls 166", + "type" : "picture", + "uploader" : "mangazeta", + "url" : str, +}, + +{ + "#url" : "https://www.sex.com/pin/55435122-ecchi/", + "#comment" : "gif", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", + "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", +}, + +{ + "#url" : "https://www.sex.com/pin/55748341/", + "#comment" : "video", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", + "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", +}, + +{ + "#url" : "https://www.sex.com/pin/55847384-very-nicely-animated/", + "#comment" : "pornhub embed", + "#category": ("", "sexcom", "pin"), + "#class" : sexcom.SexcomPinExtractor, + "#pattern" : "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", +}, + +{ + "#url" : "https://www.sex.com/pin/21241874/#related", + "#category": ("", "sexcom", "related-pin"), + "#class" : sexcom.SexcomRelatedPinExtractor, + "#count" : ">= 20", +}, + +{ + "#url" : "https://www.sex.com/user/sirjuan79/pins/", + "#category": ("", "sexcom", "pins"), + "#class" : sexcom.SexcomPinsExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.sex.com/user/ronin17/exciting-hentai/", + "#category": ("", "sexcom", "board"), + "#class" : sexcom.SexcomBoardExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.sex.com/search/pics?query=ecchi", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.sex.com/videos/hentai/", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.sex.com/pics/?sort=popular&sub=all&page=1", + "#category": ("", "sexcom", "search"), + "#class" : sexcom.SexcomSearchExtractor, +}, + +) diff --git a/test/results/simplyhentai.py b/test/results/simplyhentai.py new file mode 100644 index 00000000..561ce426 --- /dev/null +++ b/test/results/simplyhentai.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import simplyhentai +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, + "#sha1_url" : "21613585ae5ec2f69ea579e9713f536fceab5bd5", + "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", +}, + +{ + "#url" : "https://www.simply-hentai.com/notfound", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, + "#exception": exception.GalleryDLException, +}, + +{ + "#url" : "https://pokemon.simply-hentai.com/mao-friends-9bc39", + "#comment" : "custom subdomain", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, +}, + +{ + "#url" : "https://www.simply-hentai.com/vocaloid/black-magnet", + "#comment" : "www subdomain, two path segments", + "#category": ("", "simplyhentai", "gallery"), + "#class" : simplyhentai.SimplyhentaiGalleryExtractor, +}, + +{ + "#url" : "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", + "#category": ("", "simplyhentai", "image"), + "#class" : simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url" : "3d8eb55240a960134891bd77fe1df7988fcdc455", + "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", +}, + +{ + "#url" : "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", + "#category": ("", "simplyhentai", "image"), + "#class" : simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url" : "f73916527211b4a40f26568ee26cd8999f5f4f30", + "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", +}, + +{ + "#url" : "https://videos.simply-hentai.com/creamy-pie-episode-02", + "#category": ("", "simplyhentai", "video"), + "#class" : simplyhentai.SimplyhentaiVideoExtractor, + "#options" : {"verify": False}, + "#pattern" : r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", + "#count" : 1, + "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", +}, + +{ + "#url" : "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", + "#category": ("", "simplyhentai", "video"), + "#class" : simplyhentai.SimplyhentaiVideoExtractor, + "#sha1_url" : "ad9a36ae06c601b6490e3c401834b4949d947eb0", + "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", +}, + +) diff --git a/test/results/skeb.py b/test/results/skeb.py new file mode 100644 index 00000000..f956986b --- /dev/null +++ b/test/results/skeb.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import skeb + + +__tests__ = ( +{ + "#url" : "https://skeb.jp/@kanade_cocotte/works/38", + "#category": ("", "skeb", "post"), + "#class" : skeb.SkebPostExtractor, + "#count" : 2, + + "anonymous" : False, + "body" : r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", + "client" : { + "avatar_url" : r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", + "header_url" : r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", + "id" : 1196514, + "name" : str, + "screen_name": "minato_ragi", + }, + "content_category": "preview", + "creator" : { + "avatar_url" : "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", + "header_url" : "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", + "id" : 159273, + "name" : "イチノセ奏", + "screen_name": "kanade_cocotte", + }, + "file_id" : int, + "file_url" : str, + "genre" : "art", + "nsfw" : False, + "original" : { + "byte_size" : int, + "duration" : None, + "extension" : r"re:psd|png", + "frame_rate": None, + "height" : 3727, + "is_movie" : False, + "width" : 2810, + }, + "post_num" : "38", + "post_url" : "https://skeb.jp/@kanade_cocotte/works/38", + "source_body" : None, + "source_thanks" : None, + "tags" : list, + "thanks" : None, + "translated_body" : False, + "translated_thanks": None, +}, + +{ + "#url" : "https://skeb.jp/@kanade_cocotte", + "#category": ("", "skeb", "user"), + "#class" : skeb.SkebUserExtractor, + "#pattern" : r"https://skeb\.imgix\.net/uploads/origins/[\w-]+\?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&fm=webp&w=800&s=\w+", + "#range" : "1-5", +}, + +{ + "#url" : "https://skeb.jp/search?q=bunny%20tree&t=works", + "#category": ("", "skeb", "search"), + "#class" : skeb.SkebSearchExtractor, + "#count" : ">= 18", + + "search_tags": "bunny tree", +}, + +{ + "#url" : "https://skeb.jp/@user/following_creators", + "#category": ("", "skeb", "following"), + "#class" : skeb.SkebFollowingExtractor, +}, + +) diff --git a/test/results/slickpic.py b/test/results/slickpic.py new file mode 100644 index 00000000..ff1eb1ff --- /dev/null +++ b/test/results/slickpic.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import slickpic + + +__tests__ = ( +{ + "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class" : slickpic.SlickpicAlbumExtractor, + "#pattern" : r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", + "#count" : 102, + "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class" : slickpic.SlickpicAlbumExtractor, + "#range" : "34", + "#sha1_content": [ + "276eb2c902187bb177ae8013e310e1d6641fba9a", + "52b5a310587de1048030ab13a912f6a3a9cc7dab", + "cec6630e659dc72db1ee1a9a6f3b525189261988", + "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", + "22e83645fc242bc3584eca7ec982c8a53a4d8a44", + ], +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/gallery/", + "#category": ("", "slickpic", "user"), + "#class" : slickpic.SlickpicUserExtractor, + "#pattern" : slickpic.SlickpicAlbumExtractor.pattern, + "#count" : ">= 358", +}, + +{ + "#url" : "https://mattcrandall.slickpic.com/", + "#category": ("", "slickpic", "user"), + "#class" : slickpic.SlickpicUserExtractor, +}, + +) diff --git a/test/results/slideshare.py b/test/results/slideshare.py new file mode 100644 index 00000000..561f0676 --- /dev/null +++ b/test/results/slideshare.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import slideshare + + +__tests__ = ( +{ + "#url" : "https://www.slideshare.net/Slideshare/get-started-with-slide-share", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#pattern" : r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/get-started-with-slide-share-\d+-1024\.jpg\?cb=\d+", + "#count" : 19, + "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", + + "description" : "Get Started with SlideShare - A Beginngers Guide for Creators", + "likes" : int, + "presentation": "get-started-with-slide-share", + "date" : "dt:2015-05-20 17:38:21", + "title" : "Getting Started With SlideShare", + "user" : "Slideshare", + "views" : int, +}, + +{ + "#url" : "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", + "#comment" : "long title and description", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#sha1_url": "d8952260f8bec337dd809a958ec8091350393f6b", + + "title" : "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", + "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", +}, + +{ + "#url" : "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", + "#comment" : "mobile URL", + "#category": ("", "slideshare", "presentation"), + "#class" : slideshare.SlidesharePresentationExtractor, + "#sha1_url": "72c431cb1eccbb6794f608ecbbc01d52e8768159", +}, + +) diff --git a/test/results/smugloli.py b/test/results/smugloli.py new file mode 100644 index 00000000..5d044123 --- /dev/null +++ b/test/results/smugloli.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://smuglo.li/a/res/1154380.html", + "#category": ("vichan", "smugloli", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://smug.+/a/src/\d+(-\d)?\.\w+", + "#count" : ">= 18", + + "board" : "a", + "thread": "1154380", + "title" : "Mob Psycho 100 Season 3", +}, + +{ + "#url" : "https://smugloli.net/a/res/1145409.html", + "#category": ("vichan", "smugloli", "thread"), + "#class" : vichan.VichanThreadExtractor, +}, + +{ + "#url" : "https://smuglo.li/a", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://smuglo.li/a/1.html", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://smugloli.net/cute/catalog.html", + "#category": ("vichan", "smugloli", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/smugmug.py b/test/results/smugmug.py new file mode 100644 index 00000000..fd30a6b9 --- /dev/null +++ b/test/results/smugmug.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import smugmug + + +__tests__ = ( +{ + "#url" : "smugmug:album:cr4C7f", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#sha1_url": "2c2e576e47d4e9ce60b44871f08a8c66b5ebaceb", +}, + +{ + "#url" : "smugmug:album:Fb7hMs", + "#comment" : "empty", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#count" : 0, +}, + +{ + "#url" : "smugmug:album:6VRT8G", + "#comment" : "no 'User'", + "#category": ("", "smugmug", "album"), + "#class" : smugmug.SmugmugAlbumExtractor, + "#sha1_url": "c4a0f4c4bfd514b93cbdeb02b3345bf7ef6604df", +}, + +{ + "#url" : "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", + "#category": ("", "smugmug", "image"), + "#class" : smugmug.SmugmugImageExtractor, + "#sha1_url" : "e6408fd2c64e721fd146130dceb56a971ceb4259", + "#sha1_metadata": "b31a63d07c9c26eb0f79f52d60d171a98938f99b", + "#sha1_content" : "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", +}, + +{ + "#url" : "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", + "#comment" : "video", + "#category": ("", "smugmug", "image"), + "#class" : smugmug.SmugmugImageExtractor, + "#sha1_url" : "04d0ab1ff829ca7d78f5acb5548953df08e9a5ee", + "#sha1_metadata": "2b545184592c282b365fcbb7df6ca7952b8a3173", +}, + +{ + "#url" : "https://tdm.smugmug.com/Nature/Dove", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:cr4C7f$", +}, + +{ + "#url" : "https://tdm.smugmug.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : smugmug.SmugmugAlbumExtractor.pattern, + "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", +}, + +{ + "#url" : "https://www.smugmug.com/gallery/n-GLCjnD/", + "#comment" : "gallery node without owner", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:6VRT8G$", +}, + +{ + "#url" : "smugmug:www.sitkapics.com/TREES-and-TRAILS/", + "#comment" : "custom domain", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : "smugmug:album:ct8Nds$", +}, + +{ + "#url" : "smugmug:www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, + "#pattern" : r"smugmug:album:\w{6}$", + "#count" : ">= 14", +}, + +{ + "#url" : "smugmug:https://www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class" : smugmug.SmugmugPathExtractor, +}, + +) diff --git a/test/results/soundgasm.py b/test/results/soundgasm.py new file mode 100644 index 00000000..16f0a17c --- /dev/null +++ b/test/results/soundgasm.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import soundgasm + + +__tests__ = ( +{ + "#url" : "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class" : soundgasm.SoundgasmAudioExtractor, + "#pattern" : r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", + + "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", + "extension" : "m4a", + "filename" : "26cb2b23b2f2c6094b40ee3a9167271e274b570a", + "slug" : "687-Otto-von-Toontown-12822", + "title" : "687 - Otto von Toontown (12/8/22)", + "user" : "ClassWarAndPuppies2", +}, + +{ + "#url" : "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class" : soundgasm.SoundgasmAudioExtractor, +}, + +{ + "#url" : "https://soundgasm.net/u/fierce-aphrodite", + "#category": ("", "soundgasm", "user"), + "#class" : soundgasm.SoundgasmUserExtractor, + "#pattern" : r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", + "#count" : ">= 15", + + "description": str, + "extension" : "m4a", + "filename" : r"re:^[0-9a-f]{40}$", + "slug" : str, + "title" : str, + "url" : str, + "user" : "fierce-aphrodite", +}, + +) diff --git a/test/results/speakerdeck.py b/test/results/speakerdeck.py new file mode 100644 index 00000000..5e63a4cb --- /dev/null +++ b/test/results/speakerdeck.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import speakerdeck + + +__tests__ = ( +{ + "#url" : "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", + "#category": ("", "speakerdeck", "presentation"), + "#class" : speakerdeck.SpeakerdeckPresentationExtractor, + "#pattern" : r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", + "#count" : 6, + "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", +}, + +) diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py new file mode 100644 index 00000000..8fe6d8dc --- /dev/null +++ b/test/results/subscribestar.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import subscribestar +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#pattern" : r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", + "#count" : ">= 20", + + "author_id" : 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content" : str, + "date" : datetime.datetime, + "id" : int, + "num" : int, + "post_id" : int, + "type" : r"re:image|video|attachment", + "url" : str, + "?pinned" : bool, +}, + +{ + "#url" : "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#options" : {"metadata": True}, + "#range" : "1", + + "date": datetime.datetime, +}, + +{ + "#url" : "https://subscribestar.adult/kanashiipanda", + "#category": ("", "subscribestar", "user-adult"), + "#class" : subscribestar.SubscribestarUserExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.subscribestar.com/posts/102468", + "#category": ("", "subscribestar", "post"), + "#class" : subscribestar.SubscribestarPostExtractor, + "#count" : 1, + + "author_id" : 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content" : r"re:

Brand Guidelines and Assets

", + "date" : "dt:2020-05-07 12:33:00", + "extension" : "jpg", + "filename" : "8ff61299-b249-47dc-880a-cdacc9081c62", + "group" : "imgs_and_videos", + "height" : 291, + "id" : 203885, + "num" : 1, + "pinned" : False, + "post_id" : 102468, + "type" : "image", + "width" : 700, +}, + +{ + "#url" : "https://subscribestar.adult/posts/22950", + "#category": ("", "subscribestar", "post-adult"), + "#class" : subscribestar.SubscribestarPostExtractor, + "#count" : 1, + + "date": "dt:2019-04-28 07:32:00", +}, + +) diff --git a/test/results/sushiski.py b/test/results/sushiski.py new file mode 100644 index 00000000..e1bbd20a --- /dev/null +++ b/test/results/sushiski.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import misskey + + +__tests__ = ( +{ + "#url" : "https://sushi.ski/@ui@misskey.04.si", + "#category": ("misskey", "sushi.ski", "user"), + "#class" : misskey.MisskeyUserExtractor, +}, + +{ + "#url" : "https://sushi.ski/@hatusimo_sigure/following", + "#category": ("misskey", "sushi.ski", "following"), + "#class" : misskey.MisskeyFollowingExtractor, +}, + +{ + "#url" : "https://sushi.ski/notes/9bm3x4ksqw", + "#category": ("misskey", "sushi.ski", "note"), + "#class" : misskey.MisskeyNoteExtractor, + "#pattern" : r"https://media\.sushi\.ski/files/[\w-]+\.png", + "#count" : 1, +}, + +{ + "#url" : "https://sushi.ski/my/favorites", + "#category": ("misskey", "sushi.ski", "favorite"), + "#class" : misskey.MisskeyFavoriteExtractor, +}, + +) diff --git a/test/results/tapas.py b/test/results/tapas.py new file mode 100644 index 00000000..a3d79842 --- /dev/null +++ b/test/results/tapas.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tapas + + +__tests__ = ( +{ + "#url" : "https://tapas.io/series/just-leave-me-be", + "#category": ("", "tapas", "series"), + "#class" : tapas.TapasSeriesExtractor, + "#pattern" : r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", + "#count" : 132, +}, + +{ + "#url" : "https://tapas.io/series/yona", + "#comment" : "mature", + "#category": ("", "tapas", "series"), + "#class" : tapas.TapasSeriesExtractor, + "#count" : 26, +}, + +{ + "#url" : "https://tapas.io/episode/2068651", + "#category": ("", "tapas", "episode"), + "#class" : tapas.TapasEpisodeExtractor, + "#pattern" : "^text:", + "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", + + "book" : True, + "comment_cnt" : int, + "date" : "dt:2021-02-23 16:02:07", + "early_access" : False, + "escape_title" : "You are a Tomb Raider (2)", + "free" : True, + "id" : 2068651, + "like_cnt" : int, + "liked" : bool, + "mature" : False, + "next_ep_id" : 2068652, + "nsfw" : False, + "nu" : False, + "num" : 1, + "open_comments" : True, + "pending_scene" : 2, + "prev_ep_id" : 2068650, + "publish_date" : "2021-02-23T16:02:07Z", + "read" : bool, + "related_ep_id" : None, + "relative_publish_date": "Feb 23, 2021", + "scene" : 2, + "scheduled" : False, + "title" : "You are a Tomb Raider (2)", + "unlock_cnt" : 0, + "unlocked" : False, + "view_cnt" : int, + "series" : { + "genre" : dict, + "has_book_cover": True, + "has_top_banner": True, + "id" : 199931, + "premium" : True, + "sale_type" : "PAID", + "subscribed" : bool, + "thumbsup_cnt" : int, + "title" : "Tomb Raider King", + "type" : "BOOKS", + "url" : "tomb-raider-king-novel", + }, +}, + +) diff --git a/test/results/tbib.py b/test/results/tbib.py new file mode 100644 index 00000000..e4481da5 --- /dev/null +++ b/test/results/tbib.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", + "#category": ("gelbooru_v02", "tbib", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : ">= 120", +}, + +{ + "#url" : "https://tbib.org/index.php?page=favorites&s=view&id=7881", + "#category": ("gelbooru_v02", "tbib", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 3, +}, + +{ + "#url" : "https://tbib.org/index.php?page=post&s=view&id=9233957", + "#category": ("gelbooru_v02", "tbib", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_url" : "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", + "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", +}, + +) diff --git a/test/results/tcbscans.py b/test/results/tcbscans.py new file mode 100644 index 00000000..1b77a79b --- /dev/null +++ b/test/results/tcbscans.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tcbscans + + +__tests__ = ( +{ + "#url" : "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, + "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count" : 17, + + "manga" : "Chainsaw Man", + "chapter" : 108, + "chapter_minor": "", + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, + "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count" : 18, + + "manga" : "One Piece", + "chapter" : 1065, + "chapter_minor": "", + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", + "#category": ("", "tcbscans", "chapter"), + "#class" : tcbscans.TcbscansChapterExtractor, +}, + +{ + "#url" : "https://tcbscans.com/mangas/13/chainsaw-man", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, + "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, + "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://onepiecechapters.com/mangas/15/hunter-x-hunter", + "#category": ("", "tcbscans", "manga"), + "#class" : tcbscans.TcbscansMangaExtractor, +}, + +) diff --git a/test/results/tco.py b/test/results/tco.py new file mode 100644 index 00000000..37a1b106 --- /dev/null +++ b/test/results/tco.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import urlshortener +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://t.co/bCgBY8Iv5n", + "#category": ("urlshortener", "tco", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#pattern" : "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", + "#count" : 1, +}, + +{ + "#url" : "https://t.co/abcdefghij", + "#category": ("urlshortener", "tco", "link"), + "#class" : urlshortener.UrlshortenerLinkExtractor, + "#exception": exception.NotFoundError, +}, + +) diff --git a/test/results/telegraph.py b/test/results/telegraph.py new file mode 100644 index 00000000..726981b7 --- /dev/null +++ b/test/results/telegraph.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import telegraph + + +__tests__ = ( +{ + "#url" : "https://telegra.ph/Telegraph-Test-03-28", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.png", + + "author" : "mikf", + "caption" : r"re:test|", + "count" : 2, + "date" : "dt:2022-03-28 16:01:36", + "description": "Just a test", + "post_url" : "https://telegra.ph/Telegraph-Test-03-28", + "slug" : "Telegraph-Test-03-28", + "title" : "Telegra.ph Test", +}, + +{ + "#url" : "https://telegra.ph/森-03-28", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + "#count" : 1, + + "author" : "&", + "caption" : "kokiri", + "count" : 1, + "date" : "dt:2022-03-28 16:31:26", + "description" : "コキリの森", + "extension" : "jpg", + "filename" : "3ea79d23b0dd0889f215a", + "num" : 1, + "num_formatted": "1", + "post_url" : "https://telegra.ph/森-03-28", + "slug" : "森-03-28", + "title" : "\"森\"", + "url" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", +}, + +{ + "#url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", + "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", + + "author" : "Shotacon - заходи сюда", + "caption" : "", + "count" : 19, + "date" : "dt:2022-05-27 16:17:27", + "description" : "", + "num_formatted": r"re:^\d{2}$", + "post_url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "slug" : "Vsyo-o-druzyah-moej-sestricy-05-27", + "title" : "Всё о друзьях моей сестрицы", +}, + +{ + "#url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "#category": ("", "telegraph", "gallery"), + "#class" : telegraph.TelegraphGalleryExtractor, + "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", + + "author" : "cosmos", + "caption" : "", + "count" : 89, + "date" : "dt:2022-02-21 05:57:39", + "description" : "", + "num_formatted": r"re:^\d{2}$", + "post_url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "slug" : "Disharmonica---Saber-Nero-02-21", + "title" : "Disharmonica - Saber Nero", +}, + +) diff --git a/test/results/tentaclerape.py b/test/results/tentaclerape.py new file mode 100644 index 00000000..247067f6 --- /dev/null +++ b/test/results/tentaclerape.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://tentaclerape.net/post/list/comic/1", + "#category": ("shimmie2", "tentaclerape", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#pattern" : r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://tentaclerape.net/post/view/10", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", + "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", + + "extension" : "jpg", + "file_url" : "https://tentaclerape.net/./index.php?q=/image/10.jpg", + "filename" : "10", + "height" : 427, + "id" : 10, + "md5" : "945db71eeccaef82ce44b77564260c0b", + "size" : 0, + "subcategory": "post", + "tags" : "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", + "width" : 300, +}, + +{ + "#url" : "https://tentaclerape.net/post/view/91267", + "#comment" : "video", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", +}, + +) diff --git a/test/results/test.py b/test/results/test.py new file mode 100644 index 00000000..54566cc7 --- /dev/null +++ b/test/results/test.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import test + + +__tests__ = ( +{ + "#url" : "test:pixiv", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +{ + "#url" : "test:pixiv:user,favorite:0", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +{ + "#url" : "test:", + "#category": ("", "test", ""), + "#class" : test.TestExtractor, +}, + +) diff --git a/test/results/thebarchive.py b/test/results/thebarchive.py new file mode 100644 index 00000000..5fd6e226 --- /dev/null +++ b/test/results/thebarchive.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import foolfuuka + + +__tests__ = ( +{ + "#url" : "https://thebarchive.com/b/thread/739772332/", + "#category": ("foolfuuka", "thebarchive", "thread"), + "#class" : foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", +}, + +{ + "#url" : "https://thebarchive.com/b/", + "#category": ("foolfuuka", "thebarchive", "board"), + "#class" : foolfuuka.FoolfuukaBoardExtractor, +}, + +{ + "#url" : "https://thebarchive.com/_/search/text/test/", + "#category": ("foolfuuka", "thebarchive", "search"), + "#class" : foolfuuka.FoolfuukaSearchExtractor, +}, + +{ + "#url" : "https://thebarchive.com/b/gallery/9", + "#category": ("foolfuuka", "thebarchive", "gallery"), + "#class" : foolfuuka.FoolfuukaGalleryExtractor, +}, + +) diff --git a/test/results/thecollection.py b/test/results/thecollection.py new file mode 100644 index 00000000..abe47380 --- /dev/null +++ b/test/results/thecollection.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", + "#category": ("gelbooru_v01", "thecollection", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, + "#range" : "1-25", + "#count" : 25, +}, + +{ + "#url" : "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", + "#category": ("gelbooru_v01", "thecollection", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", + "#category": ("gelbooru_v01", "thecollection", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url" : "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", + "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", +}, + +) diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py new file mode 100644 index 00000000..22bc1412 --- /dev/null +++ b/test/results/toyhouse.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import toyhouse +import datetime + + +__tests__ = ( +{ + "#url" : "https://www.toyhou.se/d-floe/art", + "#category": ("", "toyhouse", "art"), + "#class" : toyhouse.ToyhouseArtExtractor, + "#pattern" : r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", + "#range" : "1-30", + "#count" : 30, + + "artists" : list, + "characters": list, + "date" : datetime.datetime, + "hash" : r"re:\w+", + "id" : r"re:\d+", + "url" : str, + "user" : "d-floe", +}, + +{ + "#url" : "https://www.toyhou.se/kroksoc/art", + "#comment" : "protected by Content Warning", + "#category": ("", "toyhouse", "art"), + "#class" : toyhouse.ToyhouseArtExtractor, + "#count" : ">= 19", +}, + +{ + "#url" : "https://toyhou.se/~images/40587320", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, + "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", + + "artists" : ["d-floe"], + "characters": ["Sumi"], + "date" : "dt:2021-10-08 01:32:47", + "extension" : "png", + "filename" : "40587320_TT1NaBUr3FLkS1p", + "hash" : "TT1NaBUr3FLkS1p", + "id" : "40587320", + "url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", +}, + +{ + "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", + "#comment" : "direct link, multiple artists", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, + + "artists" : [ + "http://aminoapps.com/p/92sf3z", + "kroksoc (Color)", + ], + "characters": ["❀Reiichi❀"], + "date" : "dt:2021-07-03 20:02:02", + "hash" : "bqhGcwcnU", + "id" : "36817425", +}, + +{ + "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + "#category": ("", "toyhouse", "image"), + "#class" : toyhouse.ToyhouseImageExtractor, +}, + +) diff --git a/test/results/tsumino.py b/test/results/tsumino.py new file mode 100644 index 00000000..cbe2c77a --- /dev/null +++ b/test/results/tsumino.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tsumino + + +__tests__ = ( +{ + "#url" : "https://www.tsumino.com/entry/40996", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, + "#pattern" : r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", + + "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", + "gallery_id": 40996, + "date" : "dt:2018-06-29 00:00:00", + "count" : 42, + "collection": "", + "artist" : ["Itou Life"], + "group" : ["Itou Life"], + "parody" : list, + "characters": list, + "tags" : list, + "type" : "Doujinshi", + "rating" : float, + "uploader" : "sehki", + "lang" : "en", + "language" : "English", + "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", +}, + +{ + "#url" : "https://www.tsumino.com/Book/Info/40996", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Read/View/45834", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Read/Index/45834", + "#category": ("", "tsumino", "gallery"), + "#class" : tsumino.TsuminoGalleryExtractor, +}, + +{ + "#url" : "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", + "#category": ("", "tsumino", "search"), + "#class" : tsumino.TsuminoSearchExtractor, + "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", + "#category": ("", "tsumino", "search"), + "#class" : tsumino.TsuminoSearchExtractor, + "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, + "#count" : ">= 3", +}, + +) diff --git a/test/results/tumblr.py b/test/results/tumblr.py new file mode 100644 index 00000000..99883578 --- /dev/null +++ b/test/results/tumblr.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tumblr +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : {"posts": "photo"}, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", + "#count" : 1, +}, + +{ + "#url" : "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "posts" : "all", + "external": True, + }, + "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+_1280\.jpg|a\.tumblr\.com/tumblr_\w+)", + "#count" : 3, +}, + +{ + "#url" : "https://mikf123-hidden.tumblr.com/", + "#comment" : "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : {"access-token": None}, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://mikf123-hidden.tumblr.com/", + "#comment" : "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "hidden", + ], +}, + +{ + "#url" : "https://mikf123-private.tumblr.com/", + "#comment" : "password protected", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "private", + ], +}, + +{ + "#url" : "https://mikf123-private-hidden.tumblr.com/", + "#comment" : "dashboard-only & password protected", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#count" : 2, + + "tags": [ + "test", + "private", + "hidden", + ], +}, + +{ + "#url" : "https://mikf123.tumblr.com/", + "#comment" : "date-min/-max/-format (#337)", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "date-min" : "201804", + "date-max" : "201805", + "date-format": "%Y%m", + }, + "#count" : 4, +}, + +{ + "#url" : "https://donttrustthetits.tumblr.com/", + "#comment" : "pagination with 'date-max' (#2191) and 'api-key'", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, + "#options" : { + "access-token": None, + "original" : False, + "date-max" : "2015-04-25T00:00:00", + "date-min" : "2015-04-01T00:00:00", + }, + "#count" : 316, +}, + +{ + "#url" : "https://demo.tumblr.com/page/2", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://demo.tumblr.com/archive", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "tumblr:http://www.b-authentique.com/", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "tumblr:www.b-authentique.com", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art", + "#category": ("", "tumblr", "user"), + "#class" : tumblr.TumblrUserExtractor, +}, + +{ + "#url" : "http://demo.tumblr.com/post/459265350", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167770226574/text-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/181022561719/quote-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167623351559/link-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167633596145/video-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/167770026604/audio-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/172687798174/photo-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://mikf123.tumblr.com/post/181022380064/chat-post", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://kichatundk.tumblr.com/post/654953419288821760", + "#comment" : "high-quality images (#1846)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, + "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", +}, + +{ + "#url" : "https://hameru-is-cool.tumblr.com/post/639261855227002880", + "#comment" : "high-quality images (#1344)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#count" : 2, + "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", +}, + +{ + "#url" : "https://mikf123.tumblr.com/image/689860196535762944", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#pattern" : r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", +}, + +{ + "#url" : "http://ziemniax.tumblr.com/post/109697912859/", + "#comment" : "HTML response (#297)", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "http://demo.tumblr.com/image/459265350", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class" : tumblr.TumblrPostExtractor, +}, + +{ + "#url" : "http://demo.tumblr.com/tagged/Times%20Square", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count" : 1, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class" : tumblr.TumblrTagExtractor, +}, + +{ + "#url" : "https://mikf123.tumblr.com/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, + "#pattern" : r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", + "#count" : 1, + + "id": 169341068404, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class" : tumblr.TumblrDayExtractor, +}, + +{ + "#url" : "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, + "#count" : 1, +}, + +{ + "#url" : "https://www.tumblr.com/blog/view/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/blog/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +{ + "#url" : "https://www.tumblr.com/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class" : tumblr.TumblrLikesExtractor, +}, + +) diff --git a/test/results/tumblrgallery.py b/test/results/tumblrgallery.py new file mode 100644 index 00000000..a1a6eeb5 --- /dev/null +++ b/test/results/tumblrgallery.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tumblrgallery + + +__tests__ = ( +{ + "#url" : "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", + "#category": ("", "tumblrgallery", "tumblrblog"), + "#class" : tumblrgallery.TumblrgalleryTumblrblogExtractor, +}, + +{ + "#url" : "https://tumblrgallery.xyz/post/405674.html", + "#category": ("", "tumblrgallery", "post"), + "#class" : tumblrgallery.TumblrgalleryPostExtractor, + "#pattern" : r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", + "#count" : 3, +}, + +{ + "#url" : "https://tumblrgallery.xyz/s.php?q=everyday-life", + "#category": ("", "tumblrgallery", "search"), + "#class" : tumblrgallery.TumblrgallerySearchExtractor, + "#pattern" : r"https://\d+\.media\.tumblr\.com/.+", + "#count" : "< 1000", +}, + +) diff --git a/test/results/turboimagehost.py b/test/results/turboimagehost.py new file mode 100644 index 00000000..3e2069fa --- /dev/null +++ b/test/results/turboimagehost.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://www.turboimagehost.com/p/39078423/test--.png.html", + "#category": ("imagehost", "turboimagehost", "image"), + "#class" : imagehosts.TurboimagehostImageExtractor, + "#sha1_url" : "b94de43612318771ced924cb5085976f13b3b90e", + "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", + "#sha1_content" : "f38b54b17cd7462e687b58d83f00fca88b1b105a", +}, + +) diff --git a/test/results/twibooru.py b/test/results/twibooru.py new file mode 100644 index 00000000..5dd0191d --- /dev/null +++ b/test/results/twibooru.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import twibooru + + +__tests__ = ( +{ + "#url" : "https://twibooru.org/1", + "#category": ("philomena", "twibooru", "post"), + "#class" : twibooru.TwibooruPostExtractor, + "#pattern" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", + + "animated" : False, + "aspect_ratio" : 1.0, + "comment_count" : int, + "created_at" : "2020-07-08T22:26:55.743Z", + "date" : "dt:2020-07-08 22:26:55", + "description" : "Why have I done this?", + "downvotes" : 0, + "duration" : 0.0, + "faves" : int, + "first_seen_at" : "2020-07-08T22:26:55.743Z", + "format" : "png", + "height" : 576, + "hidden_from_users": False, + "id" : 1, + "intensities" : dict, + "locations" : [], + "media_type" : "image", + "mime_type" : "image/png", + "name" : "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", + "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", + "processed" : True, + "representations" : dict, + "score" : int, + "sha512_hash" : "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", + "size" : 70910, + "source_url" : "", + "tag_ids" : list, + "tags" : list, + "thumbnails_generated": True, + "updated_at" : "2022-11-27T00:34:50.483Z", + "upvotes" : int, + "view_url" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "width" : 576, + "wilson_score" : float, +}, + +{ + "#url" : "https://twibooru.org/search?q=cute", + "#category": ("philomena", "twibooru", "search"), + "#class" : twibooru.TwibooruSearchExtractor, + "#range" : "40-60", + "#count" : 21, +}, + +{ + "#url" : "https://twibooru.org/tags/cute", + "#category": ("philomena", "twibooru", "search"), + "#class" : twibooru.TwibooruSearchExtractor, + "#range" : "1-20", + "#count" : 20, +}, + +{ + "#url" : "https://twibooru.org/galleries/1", + "#category": ("philomena", "twibooru", "gallery"), + "#class" : twibooru.TwibooruGalleryExtractor, + "#range" : "1-20", + + "gallery": { + "description" : "Best nation pone and russian related pics.", + "id" : 1, + "spoiler_warning": "Russia", + "thumbnail_id" : 694923, + "title" : "Marussiaverse", + }, +}, + +) diff --git a/test/results/twitter.py b/test/results/twitter.py new file mode 100644 index 00000000..4094bc94 --- /dev/null +++ b/test/results/twitter.py @@ -0,0 +1,574 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import twitter +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://twitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, + "#options" : {"include": "all"}, + "#pattern" : r"https://twitter\.com/supernaturepics/(photo|header_photo|timeline|tweets|media|with_replies|likes)$", + "#count" : 7, +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics?p=i", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/i/user/2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/intent/user?user_id=2976459548", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://fxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://vxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://x.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class" : twitter.TwitterUserExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/timeline", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://twitter.com/OptionalTypo/timeline", + "#comment" : "suspended account (#2216)", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://twitter.com/id:772949683521978368/timeline", + "#comment" : "suspended account user ID", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/timeline#t", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/timeline", + "#category": ("", "twitter", "timeline"), + "#class" : twitter.TwitterTimelineExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/tweets", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/tweets#t", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/tweets", + "#category": ("", "twitter", "tweets"), + "#class" : twitter.TwitterTweetsExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/with_replies", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/with_replies#t", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/with_replies", + "#category": ("", "twitter", "replies"), + "#class" : twitter.TwitterRepliesExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/media", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, + "#range" : "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", +}, + +{ + "#url" : "https://mobile.twitter.com/supernaturepics/media#t", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/media", + "#category": ("", "twitter", "media"), + "#class" : twitter.TwitterMediaExtractor, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/likes", + "#category": ("", "twitter", "likes"), + "#class" : twitter.TwitterLikesExtractor, +}, + +{ + "#url" : "https://twitter.com/i/bookmarks", + "#category": ("", "twitter", "bookmark"), + "#class" : twitter.TwitterBookmarkExtractor, +}, + +{ + "#url" : "https://twitter.com/i/lists/784214683683127296", + "#category": ("", "twitter", "list"), + "#class" : twitter.TwitterListExtractor, + "#range" : "1-40", + "#count" : 40, + "#archive" : False, +}, + +{ + "#url" : "https://twitter.com/i/lists/784214683683127296/members", + "#category": ("", "twitter", "list-members"), + "#class" : twitter.TwitterListMembersExtractor, + "#pattern" : twitter.TwitterTimelineExtractor.pattern, + "#range" : "1-40", + "#count" : 40, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/following", + "#category": ("", "twitter", "following"), + "#class" : twitter.TwitterFollowingExtractor, +}, + +{ + "#url" : "https://www.twitter.com/id:2976459548/following", + "#category": ("", "twitter", "following"), + "#class" : twitter.TwitterFollowingExtractor, +}, + +{ + "#url" : "https://twitter.com/search?q=nature", + "#category": ("", "twitter", "search"), + "#class" : twitter.TwitterSearchExtractor, + "#range" : "1-20", + "#count" : 20, + "#archive" : False, +}, + +{ + "#url" : "https://twitter.com/hashtag/nature", + "#category": ("", "twitter", "hashtag"), + "#class" : twitter.TwitterHashtagExtractor, + "#pattern" : twitter.TwitterSearchExtractor.pattern, + "#sha1_url": "3571c3a53b7647ea35517041fdc17f77ec5b2cb9", +}, + +{ + "#url" : "https://twitter.com/i/events/1484669206993903616", + "#category": ("", "twitter", "event"), + "#class" : twitter.TwitterEventExtractor, + "#range" : "1-20", + "#count" : ">=1", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment" : "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#sha1_url" : "88a40f7d25529c2501c46f2218f9e0de9aa634b4", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", +}, + +{ + "#url" : "https://twitter.com/perrypumas/status/894001459754180609", + "#comment" : "4 images", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", +}, + +{ + "#url" : "https://twitter.com/perrypumas/status/1065692031626829824", + "#comment" : "video", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", +}, + +{ + "#url" : "https://twitter.com/playpokemon/status/1263832915173048321", + "#comment" : "content with emoji, newlines, hashtags (#338)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! + +You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! + +❓🎁➡️ """, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1170041925560258560", + "#comment" : "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1170041925560258560", + "#comment" : "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": False}, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1424882930803908612", + "#comment" : "'replies' to self (#1254)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": "self"}, + "#count" : 4, + + "user": { + "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", + "url" : "http://princecanary.tumblr.com", + }, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1424898916156284928", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"replies": "self"}, + "#count" : 1, +}, + +{ + "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"quoted": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", + "#count" : 8, +}, + +{ + "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment" : "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#pattern" : r"https://pbs\.twimg\.com/media/EaK.+=jpg", + "#count" : 4, +}, + +{ + "#url" : "https://twitter.com/web/status/1644907989109751810", + "#comment" : "different 'user' and 'author' in quoted Tweet (#3922)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "author": { + "id" : 321629993, + "name": "Cakes_Comics", + }, + "user" : { + "id" : 718928225360080897, + "name": "StobiesGalaxy", + }, +}, + +{ + "#url" : "https://twitter.com/i/web/status/112900228289540096", + "#comment" : "TwitPic embeds (#579)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : { + "twitpic": True, + "cards" : False, + }, + "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.jpg", + "#count" : 2, +}, + +{ + "#url" : "https://twitter.com/shimoigusaP/status/8138669971", + "#comment" : "TwitPic URL not in 'urls' (#3792)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"twitpic": True}, + "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.png", + "#count" : 1, +}, + +{ + "#url" : "https://twitter.com/billboard/status/1306599586602135555", + "#comment" : "Twitter card (#1005)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs.twimg.com/card_img/\d+/", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1561674543323910144", + "#comment" : "unified_card image_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/F.+=jpg", +}, + +{ + "#url" : "https://twitter.com/doax_vv_staff/status/1479438945662685184", + "#comment" : "unified_card image_carousel_website", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs\.twimg\.com/media/F.+=png", + "#count" : 6, +}, + +{ + "#url" : "https://twitter.com/bang_dream_1242/status/1561548715348746241", + "#comment" : "unified_card video_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1466183847628865544", + "#comment" : "unified_card without type", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1571141912295243776", + "#comment" : "'cards-blacklist' option", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : { + "cards" : "ytdl", + "cards-blacklist": ("twitch.tv",), + }, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment" : "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": True}, + "#count" : 2, + + "tweet_id" : 1296304589591810048, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:34:32", + "date_original": "dt:2020-08-20 04:00:28", +}, + +{ + "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment" : "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": "original"}, + "#count" : 2, + + "tweet_id" : 1296296016002547713, + "retweet_id" : 1296296016002547713, + "date" : "dt:2020-08-20 04:00:28", + "date_original": "dt:2020-08-20 04:00:28", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment" : "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"conversations": True}, + "#count" : 5, +}, + +{ + "#url" : "https://twitter.com/morino_ya/status/1392763691599237121", + "#comment" : "retweet with missing media entities (#1555)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"retweets": True}, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1460044411165888515", + "#comment" : "deleted quote tweet (#2225)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/i/web/status/1486373748911575046", + "#comment" : "'Misleading' content", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://twitter.com/mightbecursed/status/1492954264909479936", + "#comment" : "age-restricted (#2354)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"syndication": True}, + "#count" : 1, + + "date": "dt:2022-02-13 20:10:09", +}, + +{ + "#url" : "https://twitter.com/my0nruri/status/1528379296041299968", + "#comment" : "media alt texts / descriptions (#2617)", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "description": "oc", +}, + +{ + "#url" : "https://twitter.com/poco_dandy/status/1150646424461176832", + "#comment" : "'?format=...&name=...'-style URLs", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + "#options" : {"cards": True}, + "#pattern" : r"https://pbs.twimg.com/card_img/157\d+/[\w-]+\?format=(jpg|png)&name=orig$", + "#range" : "1-2", +}, + +{ + "#url" : "https://twitter.com/i/web/status/1629193457112686592", + "#comment" : "note tweet with long 'content'", + "#category": ("", "twitter", "tweet"), + "#class" : twitter.TwitterTweetExtractor, + + "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. + +The Washington Post writes, "Three weeks after the toxic train derailment in Ohio, an analysis of Environmental Protection Agency data has found nine air pollutants at levels that could raise long-term health concerns in and around East Palestine, according to an independent analysis. + +"The analysis by Texas A&M University seems to contradict statements by state and federal regulators that air near the crash site is completely safe, despite residents complaining about rashes, breathing problems and other health effects." Your reaction.""", +}, + +{ + "#url" : "https://twitter.com/supernaturepics/photo", + "#category": ("", "twitter", "avatar"), + "#class" : twitter.TwitterAvatarExtractor, + "#pattern" : r"https://pbs\.twimg\.com/profile_images/554585280938659841/FLVAlX18\.jpeg", + + "date" : "dt:2015-01-12 10:26:49", + "extension": "jpeg", + "filename" : "FLVAlX18", + "tweet_id" : 554585280938659841, +}, + +{ + "#url" : "https://twitter.com/User16/photo", + "#category": ("", "twitter", "avatar"), + "#class" : twitter.TwitterAvatarExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://twitter.com/supernaturepics/header_photo", + "#category": ("", "twitter", "background"), + "#class" : twitter.TwitterBackgroundExtractor, + "#pattern" : r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", + + "date" : "dt:2015-01-12 10:29:43", + "filename": "1421058583", + "tweet_id": 554586009367478272, +}, + +{ + "#url" : "https://twitter.com/User16/header_photo", + "#category": ("", "twitter", "background"), + "#class" : twitter.TwitterBackgroundExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", + "#category": ("", "twitter", "image"), + "#class" : twitter.TwitterImageExtractor, + "#options" : {"size": "4096x4096,orig"}, + "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", +}, + +{ + "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", + "#category": ("", "twitter", "image"), + "#class" : twitter.TwitterImageExtractor, +}, + +) diff --git a/test/results/unique-vintage.py b/test/results/unique-vintage.py new file mode 100644 index 00000000..8fe37a3b --- /dev/null +++ b/test/results/unique-vintage.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.unique-vintage.com/collections/flapper-1920s", + "#category": ("shopify", "unique-vintage", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", + "#category": ("shopify", "unique-vintage", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/unsplash.py b/test/results/unsplash.py new file mode 100644 index 00000000..1568ed3a --- /dev/null +++ b/test/results/unsplash.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import unsplash + + +__tests__ = ( +{ + "#url" : "https://unsplash.com/photos/lsoogGC_5dg", + "#category": ("", "unsplash", "image"), + "#class" : unsplash.UnsplashImageExtractor, + "#pattern" : r"https://images\.unsplash\.com/photo-1586348943529-beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", + + "alt_description": r"re:silhouette of trees near body of water ", + "blur_hash" : "LZP4uQS4jboe%#o0WCa}2doJNaaz", + "? categories" : list, + "color" : "#f3c08c", + "created_at" : "2020-04-08T12:29:42Z", + "date" : "dt:2020-04-08 12:29:42", + "description" : "The Island", + "downloads" : int, + "exif" : { + "aperture" : "11", + "exposure_time": "30", + "focal_length" : "70.0", + "iso" : 200, + "make" : "Canon", + "model" : "Canon EOS 5D Mark IV", + }, + "extension" : "jpg", + "filename" : "photo-1586348943529-beaae6c28db9", + "height" : 6272, + "id" : "lsoogGC_5dg", + "liked_by_user" : False, + "likes" : int, + "location" : { + "city" : "Beaver Dam", + "country" : "United States", + "name" : "Beaver Dam, WI 53916, USA", + "position": { + "latitude" : 43.457769, + "longitude": -88.837329, + }, + }, + "promoted_at" : "2020-04-08T15:12:03Z", + "sponsorship" : None, + "tags" : list, + "updated_at" : str, + "user" : { + "accepted_tos" : True, + "bio" : str, + "first_name" : "Dave", + "id" : "uMJXuywXLiU", + "instagram_username": "just_midwest_rock", + "last_name" : "Hoefler", + "location" : None, + "name" : "Dave Hoefler", + "portfolio_url" : None, + "total_collections" : int, + "total_likes" : int, + "total_photos" : int, + "twitter_username" : None, + "updated_at" : str, + "username" : "davehoefler", + }, + "views" : int, + "width" : 4480, +}, + +{ + "#url" : "https://unsplash.com/@davehoefler", + "#category": ("", "unsplash", "user"), + "#class" : unsplash.UnsplashUserExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://unsplash.com/@davehoefler/likes", + "#category": ("", "unsplash", "favorite"), + "#class" : unsplash.UnsplashFavoriteExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://unsplash.com/collections/3178572/winter", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, + "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, + + "collection_id" : "3178572", + "collection_title": "winter", +}, + +{ + "#url" : "https://unsplash.com/collections/3178572/", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, +}, + +{ + "#url" : "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", + "#category": ("", "unsplash", "collection"), + "#class" : unsplash.UnsplashCollectionExtractor, +}, + +{ + "#url" : "https://unsplash.com/s/photos/hair-style", + "#category": ("", "unsplash", "search"), + "#class" : unsplash.UnsplashSearchExtractor, + "#pattern" : r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range" : "1-30", + "#count" : 30, +}, + +) diff --git a/test/results/uploadir.py b/test/results/uploadir.py new file mode 100644 index 00000000..18d6a204 --- /dev/null +++ b/test/results/uploadir.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import uploadir + + +__tests__ = ( +{ + "#url" : "https://uploadir.com/u/rd3t46ry", + "#comment" : "image", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/u/rd3t46ry", + "#count" : 1, + + "extension": "jpg", + "filename" : "Chloe and Rachel 4K jpg", + "id" : "rd3t46ry", +}, + +{ + "#url" : "https://uploadir.com/uploads/gxe8ti9v/downloads/new", + "#comment" : "archive", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/uploads/gxe8ti9v/downloads", + "#count" : 1, + + "extension": "zip", + "filename" : "NYAN-Mods-Pack#1", + "id" : "gxe8ti9v", +}, + +{ + "#url" : "https://uploadir.com/u/fllda6xl", + "#comment" : "utf-8 filename", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, + "#pattern" : r"https://uploadir\.com/u/fllda6xl", + "#count" : 1, + + "extension": "png", + "filename" : "_圖片_🖼_image_", + "id" : "fllda6xl", +}, + +{ + "#url" : "https://uploadir.com/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, +}, + +{ + "#url" : "https://uploadir.com/user/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class" : uploadir.UploadirFileExtractor, +}, + +) diff --git a/test/results/vanillarock.py b/test/results/vanillarock.py new file mode 100644 index 00000000..f0cf3e1c --- /dev/null +++ b/test/results/vanillarock.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vanillarock + + +__tests__ = ( +{ + "#url" : "https://vanilla-rock.com/mizuhashi_parsee-5", + "#category": ("", "vanillarock", "post"), + "#class" : vanillarock.VanillarockPostExtractor, + "#sha1_url" : "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", + "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", +}, + +{ + "#url" : "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", + "#category": ("", "vanillarock", "tag"), + "#class" : vanillarock.VanillarockTagExtractor, + "#pattern" : vanillarock.VanillarockPostExtractor.pattern, + "#count" : ">= 12", +}, + +{ + "#url" : "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", + "#category": ("", "vanillarock", "tag"), + "#class" : vanillarock.VanillarockTagExtractor, + "#pattern" : vanillarock.VanillarockPostExtractor.pattern, + "#count" : ">= 5", +}, + +) diff --git a/test/results/vidyart2.py b/test/results/vidyart2.py new file mode 100644 index 00000000..6c11dcbd --- /dev/null +++ b/test/results/vidyart2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v01 + + +__tests__ = ( +{ + "#url" : "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "vidyart2", "tag"), + "#class" : gelbooru_v01.GelbooruV01TagExtractor, +}, + +{ + "#url" : "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "vidyart2", "favorite"), + "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, +}, + +{ + "#url" : "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", + "#category": ("gelbooru_v01", "vidyart2", "post"), + "#class" : gelbooru_v01.GelbooruV01PostExtractor, +}, + +) diff --git a/test/results/vipergirls.py b/test/results/vipergirls.py new file mode 100644 index 00000000..7f0910d1 --- /dev/null +++ b/test/results/vipergirls.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vipergirls + + +__tests__ = ( +{ + "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, + "#count" : 225, + "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", +}, + +{ + "#url" : "https://vipergirls.to/threads/6858916-Karina/page4", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, + "#count" : 1279, +}, + +{ + "#url" : "https://vipergirls.to/threads/4328304", + "#category": ("", "vipergirls", "thread"), + "#class" : vipergirls.VipergirlsThreadExtractor, +}, + +{ + "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", + "#category": ("", "vipergirls", "post"), + "#class" : vipergirls.VipergirlsPostExtractor, + "#pattern" : r"https://vipr\.im/\w{12}$", + "#range" : "2-113", + "#count" : 112, + + "id" : "116038081", + "imagecount": "113", + "number" : "116038081", + "thread_id" : "4328304", + "title" : "FemJoy Danica - Simply Beautiful (x112) 3000x4500", +}, + +) diff --git a/test/results/vipr.py b/test/results/vipr.py new file mode 100644 index 00000000..f5e00269 --- /dev/null +++ b/test/results/vipr.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import imagehosts + + +__tests__ = ( +{ + "#url" : "https://vipr.im/kcd5jcuhgs3v.html", + "#category": ("imagehost", "vipr", "image"), + "#class" : imagehosts.ViprImageExtractor, + "#sha1_url" : "88f6a3ecbf3356a11ae0868b518c60800e070202", + "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", +}, + +) diff --git a/test/results/vk.py b/test/results/vk.py new file mode 100644 index 00000000..b681bdf9 --- /dev/null +++ b/test/results/vk.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vk +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://vk.com/id398982326", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#pattern" : r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", + "#count" : ">= 35", + + "id" : r"re:\d+", + "user": { + "id" : "398982326", + "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", + "name": "", + "nick": "Dobrov Kurva", + }, +}, + +{ + "#url" : "https://vk.com/cosplayinrussia", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#range" : "15-25", + + "id" : r"re:\d+", + "user": { + "id" : "-165740836", + "info": str, + "name": "cosplayinrussia", + "nick": "Косплей | Cosplay 18+", + }, +}, + +{ + "#url" : "https://vk.com/id76957806", + "#comment" : "photos without width/height (#2535)", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, + "#pattern" : r"https://sun\d+-\d+\.userapi\.com/", + "#range" : "1-9", + "#count" : 9, +}, + +{ + "#url" : "https://m.vk.com/albums398982326", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://www.vk.com/id398982326?profile=1", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://vk.com/albums-165740836", + "#category": ("", "vk", "photos"), + "#class" : vk.VkPhotosExtractor, +}, + +{ + "#url" : "https://vk.com/album-165740836_281339889", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#count" : 12, +}, + +{ + "#url" : "https://vk.com/album-53775183_00", + "#comment" : "'Access denied' (#2556)", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://vk.com/album232175027_00", + "#category": ("", "vk", "album"), + "#class" : vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, +}, + +{ + "#url" : "https://vk.com/tag304303884", + "#category": ("", "vk", "tagged"), + "#class" : vk.VkTaggedExtractor, + "#count" : 44, +}, + +) diff --git a/test/results/vsco.py b/test/results/vsco.py new file mode 100644 index 00000000..16f501d9 --- /dev/null +++ b/test/results/vsco.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vsco + + +__tests__ = ( +{ + "#url" : "https://vsco.co/missuri/gallery", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, + "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range" : "1-80", + "#count" : 80, +}, + +{ + "#url" : "https://vsco.co/missuri/images/1", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, +}, + +{ + "#url" : "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class" : vsco.VscoUserExtractor, +}, + +{ + "#url" : "https://vsco.co/vsco/collection/1", + "#category": ("", "vsco", "collection"), + "#class" : vsco.VscoCollectionExtractor, + "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range" : "1-80", + "#count" : 80, +}, + +{ + "#url" : "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", + "#category": ("", "vsco", "image"), + "#class" : vsco.VscoImageExtractor, + "#sha1_url" : "a45f9712325b42742324b330c348b72477996031", + "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", + + "id" : "5d34b93ef632433030707ce2", + "user" : "erenyildiz", + "grid" : "erenyildiz", + "meta" : dict, + "tags" : list, + "date" : "dt:2019-07-21 19:12:11", + "video" : False, + "width" : 1537, + "height" : 1537, + "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", +}, + +{ + "#url" : "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", + "#category": ("", "vsco", "image"), + "#class" : vsco.VscoImageExtractor, + "#sha1_url" : "08e7eef3301756ce81206c0b47c1e9373756a74a", + "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", + + "video": True, +}, + +) diff --git a/test/results/wallhaven.py b/test/results/wallhaven.py new file mode 100644 index 00000000..47a8ba77 --- /dev/null +++ b/test/results/wallhaven.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wallhaven + + +__tests__ = ( +{ + "#url" : "https://wallhaven.cc/search?q=touhou", + "#category": ("", "wallhaven", "search"), + "#class" : wallhaven.WallhavenSearchExtractor, +}, + +{ + "#url" : "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", + "#category": ("", "wallhaven", "search"), + "#class" : wallhaven.WallhavenSearchExtractor, + "#pattern" : r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#count" : "<= 30", +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/favorites/74", + "#category": ("", "wallhaven", "collection"), + "#class" : wallhaven.WallhavenCollectionExtractor, + "#count" : ">= 50", +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/", + "#category": ("", "wallhaven", "user"), + "#class" : wallhaven.WallhavenUserExtractor, +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/favorites", + "#category": ("", "wallhaven", "collections"), + "#class" : wallhaven.WallhavenCollectionsExtractor, + "#pattern" : wallhaven.WallhavenCollectionExtractor.pattern, + "#count" : 4, +}, + +{ + "#url" : "https://wallhaven.cc/user/AksumkA/uploads", + "#category": ("", "wallhaven", "uploads"), + "#class" : wallhaven.WallhavenUploadsExtractor, + "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#range" : "1-100", + "#count" : 100, +}, + +{ + "#url" : "https://wallhaven.cc/w/01w334", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, + "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", + "#sha1_content": "497212679383a465da1e35bd75873240435085a2", + + "id" : "01w334", + "width" : 1920, + "height" : 1200, + "resolution" : "1920x1200", + "ratio" : "1.6", + "colors" : list, + "tags" : list, + "file_size" : 278799, + "file_type" : "image/jpeg", + "purity" : "sfw", + "short_url" : "https://whvn.cc/01w334", + "source" : str, + "uploader" : { + "group" : "Owner/Developer", + "username": "AksumkA", + }, + "date" : "dt:2014-08-31 06:17:19", + "wh_category": "anime", + "views" : int, + "favorites" : int, +}, + +{ + "#url" : "https://wallhaven.cc/w/dge6v3", + "#comment" : "NSFW", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, + "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", +}, + +{ + "#url" : "https://whvn.cc/01w334", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, +}, + +{ + "#url" : "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", + "#category": ("", "wallhaven", "image"), + "#class" : wallhaven.WallhavenImageExtractor, +}, + +) diff --git a/test/results/wallpapercave.py b/test/results/wallpapercave.py new file mode 100644 index 00000000..f1f191ce --- /dev/null +++ b/test/results/wallpapercave.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wallpapercave + + +__tests__ = ( +{ + "#url" : "https://wallpapercave.com/w/wp10270355", + "#category": ("", "wallpapercave", "image"), + "#class" : wallpapercave.WallpapercaveImageExtractor, + "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", +}, + +) diff --git a/test/results/warosu.py b/test/results/warosu.py new file mode 100644 index 00000000..c9273de7 --- /dev/null +++ b/test/results/warosu.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import warosu + + +__tests__ = ( +{ + "#url" : "https://warosu.org/jp/thread/16656025", + "#category": ("", "warosu", "thread"), + "#class" : warosu.WarosuThreadExtractor, + "#sha1_url" : "889d57246ed67e491e5b8f7f124e50ea7991e770", + "#sha1_metadata": "c00ea4c5460c5986994f17bb8416826d42ca57c0", +}, + +{ + "#url" : "https://warosu.org/jp/thread/16658073", + "#category": ("", "warosu", "thread"), + "#class" : warosu.WarosuThreadExtractor, + "#sha1_url" : "4500cf3184b067424fd9883249bd543c905fbecd", + "#sha1_metadata": "7534edf4ec51891dbf44d775b73fbbefd52eec71", + "#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", +}, + +) diff --git a/test/results/weasyl.py b/test/results/weasyl.py new file mode 100644 index 00000000..1b41a6c5 --- /dev/null +++ b/test/results/weasyl.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import weasyl + + +__tests__ = ( +{ + "#url" : "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class" : weasyl.WeasylSubmissionExtractor, + "#pattern" : "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", + + "comments" : int, + "date" : "dt:2012-04-20 00:38:04", + "description" : """

(flex)

+""", + "favorites" : int, + "folder_name" : "Wesley Stuff", + "folderid" : 2081, + "friends_only": False, + "owner" : "Fiz", + "owner_login" : "fiz", + "rating" : "general", + "submitid" : 2031, + "subtype" : "visual", + "tags" : list, + "title" : "A Wesley!", + "type" : "submission", + "views" : int, +}, + +{ + "#url" : "https://www.weasyl.com/submission/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class" : weasyl.WeasylSubmissionExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/~tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, + "#count" : ">= 200", +}, + +{ + "#url" : "https://www.weasyl.com/submissions/tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/~aro~so", + "#category": ("", "weasyl", "submissions"), + "#class" : weasyl.WeasylSubmissionsExtractor, +}, + +{ + "#url" : "https://www.weasyl.com/submissions/tanidareal?folderid=7403", + "#category": ("", "weasyl", "folder"), + "#class" : weasyl.WeasylFolderExtractor, + "#count" : ">= 12", +}, + +{ + "#url" : "https://www.weasyl.com/journal/17647/bbcode", + "#category": ("", "weasyl", "journal"), + "#class" : weasyl.WeasylJournalExtractor, + + "title" : "BBCode", + "date" : "dt:2013-09-19 23:11:23", + "content": """

javascript:alert(42);

+ +

No more of that!

+""", +}, + +{ + "#url" : "https://www.weasyl.com/journals/charmander", + "#category": ("", "weasyl", "journals"), + "#class" : weasyl.WeasylJournalsExtractor, + "#count" : ">= 2", +}, + +{ + "#url" : "https://www.weasyl.com/favorites?userid=184616&feature=submit", + "#category": ("", "weasyl", "favorite"), + "#class" : weasyl.WeasylFavoriteExtractor, + "#count" : ">= 5", +}, + +) diff --git a/test/results/webmshare.py b/test/results/webmshare.py new file mode 100644 index 00000000..007a12b4 --- /dev/null +++ b/test/results/webmshare.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import webmshare + + +__tests__ = ( +{ + "#url" : "https://webmshare.com/O9mWY", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, + + "date" : "dt:2022-12-04 00:00:00", + "extension": "webm", + "filename" : "O9mWY", + "height" : 568, + "id" : "O9mWY", + "thumb" : "https://s1.webmshare.com/t/O9mWY.jpg", + "title" : "Yeah buddy over here", + "url" : "https://s1.webmshare.com/O9mWY.webm", + "views" : int, + "width" : 320, +}, + +{ + "#url" : "https://s1.webmshare.com/zBGAg.webm", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, + + "date" : "dt:2018-12-07 00:00:00", + "height": 1080, + "id" : "zBGAg", + "thumb" : "https://s1.webmshare.com/t/zBGAg.jpg", + "title" : "", + "url" : "https://s1.webmshare.com/zBGAg.webm", + "views" : int, + "width" : 1920, +}, + +{ + "#url" : "https://webmshare.com/play/zBGAg", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, +}, + +{ + "#url" : "https://webmshare.com/download-webm/zBGAg", + "#category": ("", "webmshare", "video"), + "#class" : webmshare.WebmshareVideoExtractor, +}, + +) diff --git a/test/results/webtoons.py b/test/results/webtoons.py new file mode 100644 index 00000000..d2a177fd --- /dev/null +++ b/test/results/webtoons.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import webtoons +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", + "#category": ("", "webtoons", "episode"), + "#class" : webtoons.WebtoonsEpisodeExtractor, + "#count" : 5, + "#sha1_url" : "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "#sha1_content": [ + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ], +}, + +{ + "#url" : "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", + "#category": ("", "webtoons", "episode"), + "#class" : webtoons.WebtoonsEpisodeExtractor, + "#exception": exception.NotFoundError, + + "comic" : "punderworld", + "description": str, + "episode" : "36", + "episode_no" : "40", + "genre" : "challenge", + "title" : r"re:^Punderworld - .+", + "title_no" : "312584", +}, + +{ + "#url" : "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", + "#comment" : "english", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#pattern" : webtoons.WebtoonsEpisodeExtractor.pattern, + "#range" : "1-15", + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=3", + "#comment" : "french", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#count" : ">= 15", +}, + +{ + "#url" : "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", + "#comment" : "(#820)", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, + "#count" : ">= 18", +}, + +{ + "#url" : "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", + "#comment" : "(#1643)", + "#category": ("", "webtoons", "comic"), + "#class" : webtoons.WebtoonsComicExtractor, +}, + +) diff --git a/test/results/weibo.py b/test/results/weibo.py new file mode 100644 index 00000000..f98b7fd9 --- /dev/null +++ b/test/results/weibo.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import weibo +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, + "#pattern" : r"^https://weibo\.com/u/1758989602\?tabtype=feed$", +}, + +{ + "#url" : "https://weibo.com/u/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://weibo.com/p/1758989602", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/profile/2314621010", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://www.weibo.com/p/1003062314621010/home", + "#category": ("", "weibo", "user"), + "#class" : weibo.WeiboUserExtractor, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=home", + "#category": ("", "weibo", "home"), + "#class" : weibo.WeiboHomeExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/zhouyuxi77?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1", + + "status": {"user": { + "id" : 7488709788, +}}, +}, + +{ + "#url" : "https://www.weibo.com/n/周于希Sally?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#range" : "1", + + "status": {"user": { + "id" : 7488709788, +}}, +}, + +{ + "#url" : "https://weibo.com/u/7500315942?tabtype=feed", + "#comment" : "deleted (#2521)", + "#category": ("", "weibo", "feed"), + "#class" : weibo.WeiboFeedExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=video", + "#category": ("", "weibo", "videos"), + "#class" : weibo.WeiboVideosExtractor, + "#pattern" : r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=newVideo", + "#category": ("", "weibo", "newvideo"), + "#class" : weibo.WeiboNewvideoExtractor, + "#pattern" : r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", + "#range" : "1-30", + "#count" : 30, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=article", + "#category": ("", "weibo", "article"), + "#class" : weibo.WeiboArticleExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://weibo.com/1758989602?tabtype=album", + "#category": ("", "weibo", "album"), + "#class" : weibo.WeiboAlbumExtractor, + "#pattern" : r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", + "#range" : "1-3", + "#count" : 3, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4323047042991618", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", + + "status": { + "count": 1, + "date" : "dt:2018-12-30 13:56:36", + }, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", +}, + +{ + "#url" : "https://m.weibo.cn/status/4268682979207023", + "#comment" : "unavailable video (#427)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://weibo.com/3314883543/Iy7fj4qVg", + "#comment" : "non-numeric status ID (#664)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/detail/4600272267522211", + "#comment" : "original retweets (#1542)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#options" : {"retweets": "original"}, + + "status": {"id": 4600167083287033}, +}, + +{ + "#url" : "https://weibo.com/5643044717/KkuDZ4jAA", + "#comment" : "type == livephoto (#2146)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://video\.weibo\.com/media/play\?livephoto=https%3A%2F%2Fus.sinaimg.cn%2F\w+\.mov", + "#range" : "2,4,6", +}, + +{ + "#url" : "https://weibo.com/1758989602/LvBhm5DiP", + "#comment" : "type == gif", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", +}, + +{ + "#url" : "https://weibo.com/2909128931/4409545658754086", + "#comment" : "missing 'playback_list' (#2792)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#count" : 10, +}, + +{ + "#url" : "https://weibo.com/1501933722/4142890299009993", + "#comment" : "empty 'playback_list' (#3301)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#pattern" : r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", + "#count" : 1, +}, + +{ + "#url" : "https://weibo.com/2427303621/MxojLlLgQ", + "#comment" : "mix_media_info (#3793)", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, + "#count" : 9, +}, + +{ + "#url" : "https://m.weibo.cn/status/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +{ + "#url" : "https://m.weibo.cn/5746766133/4339748116375525", + "#category": ("", "weibo", "status"), + "#class" : weibo.WeiboStatusExtractor, +}, + +) diff --git a/test/results/wikiart.py b/test/results/wikiart.py new file mode 100644 index 00000000..fef4bad5 --- /dev/null +++ b/test/results/wikiart.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikiart + + +__tests__ = ( +{ + "#url" : "https://www.wikiart.org/en/thomas-cole", + "#category": ("", "wikiart", "artist"), + "#class" : wikiart.WikiartArtistExtractor, + "#sha1_url" : "6844f207a5063c499fc1d5651b03127bc4fe2f73", + "#sha1_metadata": "09230b5f504697119e267349bf92487e657a7384", +}, + +{ + "#url" : "https://www.wikiart.org/en/thomas-cole/the-departure-1838", + "#category": ("", "wikiart", "image"), + "#class" : wikiart.WikiartImageExtractor, + "#sha1_url" : "976cc2545f308a650b5dbb35c29d3cee0f4673b3", + "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", +}, + +{ + "#url" : "https://www.wikiart.org/en/huang-shen/summer", + "#comment" : "no year or '-' in slug", + "#category": ("", "wikiart", "image"), + "#class" : wikiart.WikiartImageExtractor, + "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", +}, + +{ + "#url" : "https://www.wikiart.org/en/paintings-by-media/grisaille", + "#category": ("", "wikiart", "artworks"), + "#class" : wikiart.WikiartArtworksExtractor, + "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", +}, + +{ + "#url" : "https://www.wikiart.org/en/artists-by-century/12", + "#category": ("", "wikiart", "artists"), + "#class" : wikiart.WikiartArtistsExtractor, + "#pattern" : wikiart.WikiartArtistExtractor.pattern, + "#count" : ">= 8", +}, + +) diff --git a/test/results/wikieat.py b/test/results/wikieat.py new file mode 100644 index 00000000..f3c0d7f4 --- /dev/null +++ b/test/results/wikieat.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import vichan + + +__tests__ = ( +{ + "#url" : "https://wikieat.club/cel/res/25321.html", + "#category": ("vichan", "wikieat", "thread"), + "#class" : vichan.VichanThreadExtractor, + "#pattern" : r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+", + "#count" : ">= 200", +}, + +{ + "#url" : "https://wikieat.club/cel/index.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, + "#pattern" : vichan.VichanThreadExtractor.pattern, + "#count" : ">= 100", +}, + +{ + "#url" : "https://wikieat.club/cel/catalog.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +{ + "#url" : "https://wikieat.club/cel/2.html", + "#category": ("vichan", "wikieat", "board"), + "#class" : vichan.VichanBoardExtractor, +}, + +) diff --git a/test/results/wikifeet.py b/test/results/wikifeet.py new file mode 100644 index 00000000..dade42e7 --- /dev/null +++ b/test/results/wikifeet.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikifeet + + +__tests__ = ( +{ + "#url" : "https://www.wikifeet.com/Madison_Beer", + "#category": ("", "wikifeet", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", + "#count" : ">= 352", + + "celeb" : "Madison_Beer", + "celebrity" : "Madison Beer", + "birthday" : "dt:1999-03-05 00:00:00", + "birthplace": "United States", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "9 US", + "type" : "women", + "tags" : list, +}, + +{ + "#url" : "https://men.wikifeet.com/Chris_Hemsworth", + "#category": ("", "wikifeet", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", + "#count" : ">= 860", + + "celeb" : "Chris_Hemsworth", + "celebrity" : "Chris Hemsworth", + "birthday" : "dt:1983-08-11 00:00:00", + "birthplace": "Australia", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "12.5 US", + "type" : "men", + "tags" : list, +}, + +) diff --git a/test/results/wikifeetx.py b/test/results/wikifeetx.py new file mode 100644 index 00000000..a3d2be6e --- /dev/null +++ b/test/results/wikifeetx.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wikifeet + + +__tests__ = ( +{ + "#url" : "https://www.wikifeetx.com/Tifa_Quinn", + "#category": ("", "wikifeetx", "gallery"), + "#class" : wikifeet.WikifeetGalleryExtractor, + "#pattern" : r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", + "#count" : ">= 9", + + "celeb" : "Tifa_Quinn", + "celebrity" : "Tifa Quinn", + "birthday" : "[NOT SET]", + "birthplace": "United States", + "rating" : float, + "pid" : int, + "width" : int, + "height" : int, + "shoesize" : "4 US", + "type" : "women", + "tags" : list, +}, + +) diff --git a/test/results/windsorstore.py b/test/results/windsorstore.py new file mode 100644 index 00000000..c7cfd698 --- /dev/null +++ b/test/results/windsorstore.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shopify + + +__tests__ = ( +{ + "#url" : "https://www.windsorstore.com/collections/dresses-ball-gowns", + "#category": ("shopify", "windsorstore", "collection"), + "#class" : shopify.ShopifyCollectionExtractor, +}, + +{ + "#url" : "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", + "#category": ("shopify", "windsorstore", "product"), + "#class" : shopify.ShopifyProductExtractor, +}, + +) diff --git a/test/results/xbooru.py b/test/results/xbooru.py new file mode 100644 index 00000000..11160b42 --- /dev/null +++ b/test/results/xbooru.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import gelbooru_v02 + + +__tests__ = ( +{ + "#url" : "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", + "#category": ("gelbooru_v02", "xbooru", "tag"), + "#class" : gelbooru_v02.GelbooruV02TagExtractor, + "#count" : 11, +}, + +{ + "#url" : "https://xbooru.com/index.php?page=pool&s=show&id=757", + "#category": ("gelbooru_v02", "xbooru", "pool"), + "#class" : gelbooru_v02.GelbooruV02PoolExtractor, + "#count" : 5, + "#sha1_url": "ceeac56c179ec72301bd0b6add6355a138fdea01", +}, + +{ + "#url" : "https://xbooru.com/index.php?page=favorites&s=view&id=45206", + "#category": ("gelbooru_v02", "xbooru", "favorite"), + "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count" : 4, +}, + +{ + "#url" : "https://xbooru.com/index.php?page=post&s=view&id=1025649", + "#category": ("gelbooru_v02", "xbooru", "post"), + "#class" : gelbooru_v02.GelbooruV02PostExtractor, + "#pattern" : r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", + "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", +}, + +) diff --git a/test/results/xbunkr.py b/test/results/xbunkr.py new file mode 100644 index 00000000..a188d2e6 --- /dev/null +++ b/test/results/xbunkr.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lolisafe + + +__tests__ = ( +{ + "#url" : "https://xbunkr.com/a/TA0bu3F4", + "#category": ("lolisafe", "xbunkr", "album"), + "#class" : lolisafe.LolisafeAlbumExtractor, + "#pattern" : r"https://media\.xbunkr\.com/[^.]+\.\w+", + "#count" : 861, + + "album_id" : "TA0bu3F4", + "album_name": "Hannahowo Onlyfans Photos", +}, + +{ + "#url" : "https://xbunkr.com/a/GNQc2I5d", + "#category": ("lolisafe", "xbunkr", "album"), + "#class" : lolisafe.LolisafeAlbumExtractor, +}, + +) diff --git a/test/results/xhamster.py b/test/results/xhamster.py new file mode 100644 index 00000000..93436d06 --- /dev/null +++ b/test/results/xhamster.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import xhamster + + +__tests__ = ( +{ + "#url" : "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, + "#pattern" : r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", + "#count" : ">= 144", + + "comments": int, + "count" : int, + "favorite": bool, + "id" : int, + "num" : int, + "height" : int, + "width" : int, + "imageURL": str, + "pageURL" : str, + "thumbURL": str, + "gallery" : { + "date" : "dt:2019-04-16 00:07:31", + "description": "", + "dislikes" : int, + "id" : 11748968, + "likes" : int, + "tags" : ["NON-Porn"], + "thumbnail" : str, + "title" : "Make the world better.", + "views" : int, + }, + "user" : { + "id" : 16874672, + "name" : "Anonymousrants", + "retired" : bool, + "subscribers": int, + "url" : "https://xhamster.com/users/anonymousrants", + "verified" : bool, + }, +}, + +{ + "#url" : "https://jp.xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, + "#pattern" : r"https://thumb-p\d+.xhcdn.com/./[\w/-]+_1000.jpg$", + "#count" : ">= 144", +}, + +{ + "#url" : "https://xhamster.com/photos/gallery/make-the-world-better-11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.one/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.desi/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://en.xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.porncache.net/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class" : xhamster.XhamsterGalleryExtractor, +}, + +{ + "#url" : "https://xhamster.com/users/goldenpalomino/photos", + "#category": ("", "xhamster", "user"), + "#class" : xhamster.XhamsterUserExtractor, + "#pattern" : xhamster.XhamsterGalleryExtractor.pattern, + "#range" : "1-50", + "#count" : 50, +}, + +{ + "#url" : "https://xhamster.com/users/nickname68", + "#category": ("", "xhamster", "user"), + "#class" : xhamster.XhamsterUserExtractor, +}, + +) diff --git a/test/results/xvideos.py b/test/results/xvideos.py new file mode 100644 index 00000000..d7a682c0 --- /dev/null +++ b/test/results/xvideos.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import xvideos + + +__tests__ = ( +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, + "#pattern" : r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", + "#count" : 8, + + "gallery": { + "id" : 751031, + "title": "Random Stuff", + "tags" : list, + }, + "user" : { + "id" : 20245371, + "name" : "pervertedcouple", + "display" : "Pervertedcouple", + "sex" : "Woman", + "description": str, + }, +}, + +{ + "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, +}, + +{ + "#url" : "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class" : xvideos.XvideosGalleryExtractor, +}, + +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class" : xvideos.XvideosUserExtractor, + "#sha1_url" : "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", + "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", +}, + +{ + "#url" : "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", + "#category": ("", "xvideos", "user"), + "#class" : xvideos.XvideosUserExtractor, +}, + +) diff --git a/test/results/yandere.py b/test/results/yandere.py new file mode 100644 index 00000000..3b4d9a96 --- /dev/null +++ b/test/results/yandere.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import moebooru + + +__tests__ = ( +{ + "#url" : "https://yande.re/post/show/51824", + "#category": ("moebooru", "yandere", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"tags": True}, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + + "tags_artist" : "sasaki_tamaru", + "tags_circle" : "softhouse_chara", + "tags_copyright": "ouzoku", + "tags_general" : str, +}, + +{ + "#url" : "https://yande.re/post/show/993156", + "#category": ("moebooru", "yandere", "post"), + "#class" : moebooru.MoebooruPostExtractor, + "#options" : {"notes": True}, + "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", + + "notes": [ + { + "id" : 7096, + "x" : 90, + "y" : 626, + "width" : 283, + "height": 529, + "body" : "Please keep this as a secret for me!!", + }, + { + "id" : 7095, + "x" : 900, + "y" : 438, + "width" : 314, + "height": 588, + "body" : "The facts that I love playing games", + }, + ], +}, + +{ + "#url" : "https://yande.re/post?tags=ouzoku+armor", + "#category": ("moebooru", "yandere", "tag"), + "#class" : moebooru.MoebooruTagExtractor, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", +}, + +{ + "#url" : "https://yande.re/pool/show/318", + "#category": ("moebooru", "yandere", "pool"), + "#class" : moebooru.MoebooruPoolExtractor, + "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", +}, + +{ + "#url" : "https://yande.re/post/popular_by_month?month=6&year=2014", + "#category": ("moebooru", "yandere", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, + "#count" : 40, +}, + +{ + "#url" : "https://yande.re/post/popular_recent", + "#category": ("moebooru", "yandere", "popular"), + "#class" : moebooru.MoebooruPopularExtractor, +}, + +) diff --git a/test/results/ytdl.py b/test/results/ytdl.py new file mode 100644 index 00000000..1aecee91 --- /dev/null +++ b/test/results/ytdl.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import ytdl + + +__tests__ = ( +{ + "#url" : "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", + "#category": ("", "ytdl", "Youtube"), + "#class" : ytdl.YoutubeDLExtractor, +}, + +) diff --git a/test/results/zerochan.py b/test/results/zerochan.py new file mode 100644 index 00000000..67b9da21 --- /dev/null +++ b/test/results/zerochan.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import zerochan + + +__tests__ = ( +{ + "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class" : zerochan.ZerochanTagExtractor, + "#pattern" : r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "#count" : "> 24", + + "extension" : r"re:jpg|png", + "file_url" : r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "filename" : r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", + "height" : r"re:^\d+$", + "id" : r"re:^\d+$", + "name" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "search_tags": "Perth (Kantai Collection)", + "size" : r"re:^\d+k$", + "width" : r"re:^\d+$", +}, + +{ + "#url" : "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class" : zerochan.ZerochanImageExtractor, + "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + + "author" : "YeFan 葉凡", + "date" : "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height" : 1366, + "id" : 2920445, + "path" : [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size" : 1975296, + "tags" : [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Blonde Hair", + "Theme:Braids", + "Theme:Coat", + "Theme:Female", + "Theme:Firefighter Outfit", + "Theme:Group", + "Theme:Long Sleeves", + "Theme:Personification", + "Theme:Pins", + "Theme:Ribbon", + "Theme:Shirt", + "Theme:Short Hair", + ], + "uploader": "YukinoTokisaki", + "width" : 1920, +}, + +) diff --git a/test/test_extractor.py b/test/test_extractor.py index fb164af4..9387f5b6 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -93,20 +93,24 @@ class TestExtractorModule(unittest.TestCase): FakeExtractor.from_url(invalid) def test_unique_pattern_matches(self): - test_urls = [] + try: + import test.results + except ImportError: + raise unittest.SkipTest("no test data") # collect testcase URLs + test_urls = [] append = test_urls.append - for extr in extractor.extractors(): - for testcase in extr._get_tests(): - append((testcase[0], extr)) + + for result in test.results.all(): + append((result["#url"], result["#class"])) # iterate over all testcase URLs for url, extr1 in test_urls: matches = [] # ... and apply all regex patterns to each one - for extr2 in extractor._cache: + for extr2 in _list_classes(): # skip DirectlinkExtractor pattern if it isn't tested if extr1 != DirectlinkExtractor and \ @@ -133,15 +137,13 @@ class TestExtractorModule(unittest.TestCase): self.assertIs(extr1, matches[0][1], url) def test_init(self): - """Test for exceptions in Extractor.initialize(()""" + """Test for exceptions in Extractor.initialize() and .finalize()""" for cls in extractor.extractors(): if cls.category == "ytdl": continue - for test in cls._get_tests(): - extr = cls.from_url(test[0]) - extr.initialize() - extr.finalize() - break + extr = cls.from_url(cls.example) + extr.initialize() + extr.finalize() @unittest.skipIf(sys.hexversion < 0x3060000, "test fails in CI") def test_init_ytdl(self): diff --git a/test/test_formatter.py b/test/test_formatter.py index 0992f4ba..dbdccba1 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -340,6 +340,8 @@ class TestFormatter(unittest.TestCase): self._run_test("{'foobar'[:3]}", value) self._run_test("{z|'foo'}" , value) self._run_test("{z|''|'foo'}" , value) + self._run_test("{z|''}" , "") + self._run_test("{''|''}" , "") self._run_test("{_lit[foo]}" , value) self._run_test("{_lit[foo]!u}" , value.upper()) diff --git a/test/test_results.py b/test/test_results.py index f434b2fc..8644f275 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -15,10 +15,12 @@ import re import json import hashlib import datetime +import collections sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import \ extractor, util, job, config, exception, formatter # noqa E402 +from test import results # noqa E402 # temporary issues, etc. @@ -46,28 +48,33 @@ class TestExtractorResults(unittest.TestCase): for url, exc in cls._skipped: print('- {} ("{}")'.format(url, exc)) - def _run_test(self, extr, url, result): - if result: - if "options" in result: - for key, value in result["options"]: + def _run_test(self, result): + result.pop("#comment", None) + only_matching = (len(result) <= 3) + + if only_matching: + content = False + else: + if "#options" in result: + for key, value in result["#options"].items(): key = key.split(".") config.set(key[:-1], key[-1], value) - if "range" in result: - config.set((), "image-range" , result["range"]) - config.set((), "chapter-range", result["range"]) - content = "content" in result - else: - content = False + if "#range" in result: + config.set((), "image-range" , result["#range"]) + config.set((), "chapter-range", result["#range"]) + content = ("#sha1_content" in result) - tjob = ResultJob(url, content=content) - self.assertEqual(extr, tjob.extractor.__class__) + tjob = ResultJob(result["#url"], content=content) + self.assertEqual(result["#class"], tjob.extractor.__class__) - if not result: + if only_matching: return - if "exception" in result: - with self.assertRaises(result["exception"]): + + if "#exception" in result: + with self.assertRaises(result["#exception"]): tjob.run() return + try: tjob.run() except exception.StopExtraction: @@ -76,11 +83,11 @@ class TestExtractorResults(unittest.TestCase): exc = str(exc) if re.match(r"'5\d\d ", exc) or \ re.search(r"\bRead timed out\b", exc): - self._skipped.append((url, exc)) + self._skipped.append((result["#url"], exc)) self.skipTest(exc) raise - if result.get("archive", True): + if result.get("#archive", True): self.assertEqual( len(set(tjob.archive_list)), len(tjob.archive_list), @@ -92,7 +99,7 @@ class TestExtractorResults(unittest.TestCase): for url, kwdict in zip(tjob.url_list, tjob.kwdict_list): if "_extractor" in kwdict: extr = kwdict["_extractor"].from_url(url) - if extr is None and not result.get("extractor", True): + if extr is None and not result.get("#extractor", True): continue self.assertIsInstance(extr, kwdict["_extractor"]) self.assertEqual(extr.url, url) @@ -102,27 +109,26 @@ class TestExtractorResults(unittest.TestCase): self.assertIn("extension", kwdict) # test extraction results - if "url" in result: - self.assertEqual(result["url"], tjob.url_hash.hexdigest()) + if "#sha1_url" in result: + self.assertEqual( + result["#sha1_url"], tjob.url_hash.hexdigest()) - if "content" in result: - expected = result["content"] + if "#sha1_content" in result: + expected = result["#sha1_content"] digest = tjob.content_hash.hexdigest() if isinstance(expected, str): - self.assertEqual(digest, expected, "content") - else: # assume iterable - self.assertIn(digest, expected, "content") - - if "keyword" in result: - expected = result["keyword"] - if isinstance(expected, dict): - for kwdict in tjob.kwdict_list: - self._test_kwdict(kwdict, expected) - else: # assume SHA1 hash - self.assertEqual(expected, tjob.kwdict_hash.hexdigest()) - - if "count" in result: - count = result["count"] + self.assertEqual( + expected, digest, "content") + else: # iterable + self.assertIn( + digest, expected, "content") + + if "#sha1_metadata" in result: + self.assertEqual( + result["#sha1_metadata"], tjob.kwdict_hash.hexdigest()) + + if "#count" in result: + count = result["#count"] if isinstance(count, str): self.assertRegex(count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$") expr = "{} {}".format(len(tjob.url_list), count) @@ -130,10 +136,15 @@ class TestExtractorResults(unittest.TestCase): else: # assume integer self.assertEqual(len(tjob.url_list), count) - if "pattern" in result: + if "#pattern" in result: self.assertGreater(len(tjob.url_list), 0) for url in tjob.url_list: - self.assertRegex(url, result["pattern"]) + self.assertRegex(url, result["#pattern"]) + + metadata = {k: v for k, v in result.items() if k[0] != "#"} + if metadata: + for kwdict in tjob.kwdict_list: + self._test_kwdict(kwdict, metadata) def _test_kwdict(self, kwdict, tests): for key, test in tests.items(): @@ -326,7 +337,7 @@ def setup_test_config(): "e621", "e926", "e6ai", "instagram", "twitter", "subscribestar", "deviantart", "inkbunny", "tapas", "pillowfort", "mangadex", - "vipergirls", "gfycat"): + "vipergirls"): config.set(("extractor", category), "username", None) config.set(("extractor", "mastodon.social"), "access-token", @@ -352,39 +363,34 @@ def setup_test_config(): def generate_tests(): """Dynamically generate extractor unittests""" - def _generate_test(extr, tcase): + def _generate_method(result): def test(self): - url, result = tcase - print("\n", url, sep="") - self._run_test(extr, url, result) + print("\n" + result["#url"]) + self._run_test(result) return test # enable selective testing for direct calls - if __name__ == '__main__' and len(sys.argv) > 1: - categories = sys.argv[1:] - negate = False - if categories[0].lower() == "all": - categories = () - negate = True - elif categories[0].lower() == "broken": - categories = BROKEN + if __name__ == "__main__" and len(sys.argv) > 1: + category, _, subcategory = sys.argv[1].partition(":") del sys.argv[1:] + + tests = results.category(category) + if subcategory: + tests = [t for t in tests if t["#category"][-1] == subcategory] else: - categories = BROKEN - negate = True - if categories: - print("skipping:", ", ".join(categories)) - fltr = util.build_extractor_filter(categories, negate=negate) + tests = results.all() # add 'test_...' methods - for extr in filter(fltr, extractor.extractors()): - name = "test_" + extr.__name__ + "_" - for num, tcase in enumerate(extr._get_tests(), 1): - test = _generate_test(extr, tcase) - test.__name__ = name + str(num) - setattr(TestExtractorResults, test.__name__, test) + enum = collections.defaultdict(int) + for result in tests: + name = "{1}_{2}".format(*result["#category"]) + enum[name] += 1 + + method = _generate_method(result) + method.__name__ = "test_{}_{}".format(name, enum[name]) + setattr(TestExtractorResults, method.__name__, method) generate_tests() -if __name__ == '__main__': - unittest.main(warnings='ignore') +if __name__ == "__main__": + unittest.main(warnings="ignore")