From 625e94fa7dac9d8ccd274d83be0661675e3f6bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 27 Nov 2023 18:30:53 +0100 Subject: [PATCH] update extractor test results still not everything, but good enough for now --- test/results/2chan.py | 12 +-- test/results/4chan.py | 2 +- test/results/4plebs.py | 3 +- test/results/8chan.py | 2 +- test/results/aibooru.py | 4 +- test/results/bbc.py | 2 +- test/results/bcbnsfw.py | 3 +- test/results/cavemanon.py | 2 +- test/results/comicvine.py | 2 +- test/results/cyberdrop.py | 60 ++++++++---- test/results/derpibooru.py | 2 +- test/results/e621.py | 66 +++++++------ test/results/horne.py | 2 +- test/results/idolcomplex.py | 47 +++++++-- test/results/imagefap.py | 9 +- test/results/issuu.py | 1 - test/results/itaku.py | 21 ++-- test/results/itchio.py | 6 +- test/results/khinsider.py | 2 +- test/results/lesbianenergy.py | 2 +- test/results/luscious.py | 8 +- test/results/mangadex.py | 2 +- test/results/misskeyio.py | 2 +- test/results/myhentaigallery.py | 2 +- test/results/nsfwalbum.py | 20 +++- test/results/paheal.py | 6 +- test/results/pillowfort.py | 2 +- test/results/pornhub.py | 30 ++++-- test/results/pornpics.py | 5 +- test/results/pururin.py | 2 +- test/results/reddit.py | 2 +- test/results/sexcom.py | 3 +- test/results/skeb.py | 2 +- test/results/smugloli.py | 8 +- test/results/tapas.py | 4 +- test/results/toyhouse.py | 2 +- test/results/tumblr.py | 6 +- test/results/turboimagehost.py | 5 +- test/results/twibooru.py | 2 +- test/results/unsplash.py | 163 +++++++++++++++++++++----------- test/results/weibo.py | 1 + test/results/wikiart.py | 62 +++++++++++- test/results/wikifeet.py | 2 +- 43 files changed, 396 insertions(+), 195 deletions(-) diff --git a/test/results/2chan.py b/test/results/2chan.py index 0a1cdd69..78f48e58 100644 --- a/test/results/2chan.py +++ b/test/results/2chan.py @@ -10,25 +10,25 @@ _2chan = getattr(gallery_dl.extractor, "2chan") __tests__ = ( { - "#url" : "https://dec.2chan.net/70/res/14565.htm", + "#url" : "https://dec.2chan.net/70/res/17222.htm", "#category": ("", "2chan", "thread"), "#class" : _2chan._2chanThreadExtractor, "#pattern" : r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", - "#count" : ">= 3", + "#count" : ">= 2", "board" : "70", "board_name": "新板提案", "com" : str, "fsize" : r"re:\d+", "name" : "名無し", - "no" : r"re:1[45]\d\d\d", - "now" : r"re:22/../..\(.\)..:..:..", + "no" : r"re:17\d\d\d", + "now" : r"re:23/../..\(.\)..:..:..", "post" : "無題", "server" : "dec", - "thread" : "14565", + "thread" : "17222", "tim" : r"re:^\d{13}$", "time" : r"re:^\d{10}$", - "title" : "ﾋﾛｱｶ板", + "title" : "画像会話板", }, ) diff --git a/test/results/4chan.py b/test/results/4chan.py index c90e4107..6219e165 100644 --- a/test/results/4chan.py +++ b/test/results/4chan.py @@ -15,7 +15,7 @@ __tests__ = ( "#class" : _4chan._4chanThreadExtractor, "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - "#sha1_content" : "20b7b51afa51c9c31a0020a0737b889532c8d7ec", + "#sha1_content" : "551e432d52700ff3711f14752124e9af86ecbbdf", }, { diff --git a/test/results/4plebs.py b/test/results/4plebs.py index 4f00027d..bae62608 100644 --- a/test/results/4plebs.py +++ b/test/results/4plebs.py @@ -12,7 +12,8 @@ __tests__ = ( "#url" : "https://archive.4plebs.org/tg/thread/54059290", "#category": ("foolfuuka", "4plebs", "thread"), "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "fd823f17b5001442b941fddcd9ec91bafedfbc79", + "#pattern" : "https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)", + "#count" : 30, }, { diff --git a/test/results/8chan.py b/test/results/8chan.py index 43c08349..f7be8148 100644 --- a/test/results/8chan.py +++ b/test/results/8chan.py @@ -73,7 +73,7 @@ __tests__ = ( "#category": ("", "8chan", "board"), "#class" : _8chan._8chanBoardExtractor, "#pattern" : _8chan._8chanThreadExtractor.pattern, - "#count" : 23, + "#count" : 27, }, { diff --git a/test/results/aibooru.py b/test/results/aibooru.py index 78bd7273..41408423 100644 --- a/test/results/aibooru.py +++ b/test/results/aibooru.py @@ -12,8 +12,8 @@ __tests__ = ( "#url" : "https://aibooru.online/posts?tags=center_frills&z=1", "#category": ("Danbooru", "aibooru", "tag"), "#class" : danbooru.DanbooruTagExtractor, - "#pattern" : r"https://cdn\.aibooru\.online/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "#count" : ">= 3", + "#pattern" : r"https://cdn\.aibooru\.download/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count" : ">= 50", }, { diff --git a/test/results/bbc.py b/test/results/bbc.py index 83b4923c..e458a06a 100644 --- a/test/results/bbc.py +++ b/test/results/bbc.py @@ -39,7 +39,7 @@ __tests__ = ( }, { - "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", + "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=25", "#category": ("", "bbc", "programme"), "#class" : bbc.BbcProgrammeExtractor, "#pattern" : bbc.BbcGalleryExtractor.pattern, diff --git a/test/results/bcbnsfw.py b/test/results/bcbnsfw.py index 8d35e36e..e9fcf8b2 100644 --- a/test/results/bcbnsfw.py +++ b/test/results/bcbnsfw.py @@ -16,10 +16,11 @@ __tests__ = ( { "#url" : "https://booru.bcbnsfw.space/post/1599", + "#comment" : "now only available as WebP", "#category": ("szurubooru", "bcbnsfw", "post"), "#class" : szurubooru.SzurubooruPostExtractor, "#pattern" : r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", - "#sha1_content": "0c38fc612ba1f03950fad31c4f80a1fccdab1096", + "#sha1_content": "55f8b8d187adc82f2dcaf2aa89db0ae21b08c0b0", }, ) diff --git a/test/results/cavemanon.py b/test/results/cavemanon.py index 4fc0af3c..3d065f43 100644 --- a/test/results/cavemanon.py +++ b/test/results/cavemanon.py @@ -37,7 +37,7 @@ __tests__ = ( "id" : 8335, "md5" : "", "size" : 0, - "tags" : "Color Fang Food Pterodactyl discord_emote transparent", + "tags" : "Color discord_emote Fang Food Pterodactyl transparent", "width" : 459, }, diff --git a/test/results/comicvine.py b/test/results/comicvine.py index fa77c491..b41c24e8 100644 --- a/test/results/comicvine.py +++ b/test/results/comicvine.py @@ -21,7 +21,7 @@ __tests__ = ( "#category": ("", "comicvine", "tag"), "#class" : comicvine.ComicvineTagExtractor, "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", - "#count" : ">= 450", + "#count" : ">= 400", }, ) diff --git a/test/results/cyberdrop.py b/test/results/cyberdrop.py index b163ca47..a47d244d 100644 --- a/test/results/cyberdrop.py +++ b/test/results/cyberdrop.py @@ -9,33 +9,53 @@ from gallery_dl.extractor import cyberdrop __tests__ = ( { - "#url" : "https://cyberdrop.me/a/keKRjm4t", - "#comment" : "images", + "#url" : "https://cyberdrop.me/a/8uE0wQiK", "#category": ("lolisafe", "cyberdrop", "album"), "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$", - - "album_id" : "keKRjm4t", - "album_name" : "Fate (SFW)", - "album_size" : 150069254, - "count" : 62, - "date" : "dt:2020-06-18 13:14:20", - "description": "", - "id" : r"re:\w{8}", + "#pattern" : r"https://sun\.cyberdrop\.ch/api/fc/yyK9y8xpQK5dP\?.+", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + + "album_id" : "8uE0wQiK", + "album_name" : "test テスト \"&>", + "album_size" : 182, + "count" : 1, + "date" : "dt:2023-11-26 00:00:00", + "description" : "test テスト \"&>", + "extension" : "png", + "filename" : "test-テスト--22->-rwU3x9LU", + "id" : "rwU3x9LU", + "name" : "test-テスト--22->", + "num" : 1, + "size" : 182, + "slug" : "yyK9y8xpQK5dP", + "thumbnailUrl": str, + "type" : "image/png", + "url" : str, }, { - "#url" : "https://cyberdrop.to/a/l8gIAXVD", - "#comment" : "videos", + "#url" : "https://cyberdrop.me/a/HriMgbuf", "#category": ("lolisafe", "cyberdrop", "album"), "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$", - "#count" : 31, - - "album_id" : "l8gIAXVD", - "album_name": "Achelois17 videos", - "album_size": 652037121, - "date" : "dt:2020-06-16 15:40:44", + "#pattern" : r"https://sun\.cyberdrop\.ch/api/fc/\w+\?.+", + "#count" : 3, + + "album_id" : "HriMgbuf", + "album_name" : "animations", + "album_size" : 1090519, + "count" : 3, + "date" : "dt:2023-11-26 00:00:00", + "description" : "animated stuff", + "extension" : r"re:gif|webm", + "filename" : r"re:danbooru_\d+_\w+-\w+", + "id" : str, + "name" : r"re:danbooru_\d+_\w+", + "num" : range(1, 3), + "size" : int, + "slug" : str, + "thumbnailUrl": str, + "type" : r"re:image/gif|video/webm", + "url" : str, }, ) diff --git a/test/results/derpibooru.py b/test/results/derpibooru.py index 7ca6a0de..278fcaa0 100644 --- a/test/results/derpibooru.py +++ b/test/results/derpibooru.py @@ -40,7 +40,7 @@ __tests__ = ( "score" : int, "sha512_hash" : "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", "size" : 860914, - "source_url" : "https://www.deviantart.com/speccysy/art/Afternoon-Flight-215193985", + "source_url" : "https://web.archive.org/web/20110702164313/http://speccysy.deviantart.com:80/art/Afternoon-Flight-215193985", "spoilered" : False, "tag_count" : int, "tag_ids" : list, diff --git a/test/results/e621.py b/test/results/e621.py index dd9787cd..5cd5c74d 100644 --- a/test/results/e621.py +++ b/test/results/e621.py @@ -59,36 +59,42 @@ __tests__ = ( "#options" : {"metadata": "notes,pools"}, "#pattern" : r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", - "notes": [{ - "body" : "Little Legends 2", - "created_at" : "2022-05-16T13:58:38.877-04:00", - "creator_id" : 517450, - "creator_name": "EeveeCuddler69", - "height" : 475, - "id" : 321296, - "is_active" : True, - "post_id" : 3181052, - "updated_at" : "2022-05-16T13:59:02.050-04:00", - "version" : 3, - "width" : 809, - "x" : 83, - "y" : 117, -}], - "pools": [{ - "category" : "series", - "created_at" : "2022-02-17T00:29:22.669-05:00", - "creator_id" : 1077440, - "creator_name": "Yeetus90", - "description" : """* "Little Legends":/pools/27971 -* Little Legends 2 -* "Little Legends 3":/pools/27481""", - "id" : 27492, - "is_active" : False, - "name" : "Little Legends 2", - "post_count" : 39, - "post_ids" : list, - "updated_at" : "2022-03-27T06:30:03.382-04:00", -}], + "notes": [ + { + "body" : "Little Legends 2", + "created_at" : "2022-05-16T13:58:38.877-04:00", + "creator_id" : 517450, + "creator_name": "EeveeCuddler69", + "height" : 475, + "id" : 321296, + "is_active" : True, + "post_id" : 3181052, + "updated_at" : "2022-05-16T13:59:02.050-04:00", + "version" : 3, + "width" : 809, + "x" : 83, + "y" : 117, + }, + ], + "pools": [ + { + "category" : "series", + "created_at" : "2022-02-17T00:29:22.669-05:00", + "creator_id" : 1077440, + "creator_name": "Yeetus90", + "description" : """\ +* "Little Legends":/pools/27971\r +* Little Legends 2\r +* "Little Legends 3":/pools/27481\ +""", + "id" : 27492, + "is_active" : False, + "name" : "Little Legends 2", + "post_count" : 39, + "post_ids" : list, + "updated_at" : "2022-03-27T06:30:03.382-04:00", + }, + ], }, { diff --git a/test/results/horne.py b/test/results/horne.py index 1cea3a0e..8cbce012 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -19,7 +19,7 @@ __tests__ = ( "#url" : "https://horne.red/members_illust.php?id=58000", "#category": ("Nijie", "horne", "illustration"), "#class" : nijie.NijieIllustrationExtractor, - "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", + "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\w+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", "#range" : "1-20", "#count" : 20, diff --git a/test/results/idolcomplex.py b/test/results/idolcomplex.py index 6aced68c..5932dc72 100644 --- a/test/results/idolcomplex.py +++ b/test/results/idolcomplex.py @@ -12,37 +12,66 @@ __tests__ = ( "#url" : "https://idol.sankakucomplex.com/?tags=lyumos", "#category": ("booru", "idolcomplex", "tag"), "#class" : idolcomplex.IdolcomplexTagExtractor, - "#pattern" : r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", + "#pattern" : r"https://i[sv]\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", "#range" : "18-22", "#count" : 5, }, { - "#url" : "https://idol.sankakucomplex.com/?tags=order:favcount", + "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", "#category": ("booru", "idolcomplex", "tag"), "#class" : idolcomplex.IdolcomplexTagExtractor, - "#range" : "18-22", - "#count" : 5, }, { - "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, + "#url" : "https://idol.sankakucomplex.com/pools/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class" : idolcomplex.IdolcomplexPoolExtractor, + "#count" : 3, }, { "#url" : "https://idol.sankakucomplex.com/pool/show/145", "#category": ("booru", "idolcomplex", "pool"), "#class" : idolcomplex.IdolcomplexPoolExtractor, - "#count" : 3, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/en/posts/show/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class" : idolcomplex.IdolcomplexPostExtractor, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + + "created_at" : "2017-11-24 17:01:27.696", + "date" : "dt:2017-11-24 17:01:27", + "extension" : "jpg", + "file_url" : r"re:https://is\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?", + "filename" : "509eccbba54a43cea6b275a65b93c51d", + "height" : 683, + "id" : 694215, + "md5" : "509eccbba54a43cea6b275a65b93c51d", + "rating" : "g", + "tags" : "lyumos the_witcher shani_(the_witcher) 1girl cosplay green_eyes non-asian redhead waistcoat wreath 3:2_aspect_ratio", + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_general" : "1girl cosplay green_eyes non-asian redhead waistcoat wreath", + "tags_idol" : "lyumos", + "tags_medium" : "3:2_aspect_ratio", + "vote_average" : range(4, 5), + "vote_count" : range(25, 40), + "width" : 1024, +}, + +{ + "#url" : "https://idol.sankakucomplex.com/posts/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class" : idolcomplex.IdolcomplexPostExtractor, }, { "#url" : "https://idol.sankakucomplex.com/post/show/694215", "#category": ("booru", "idolcomplex", "post"), "#class" : idolcomplex.IdolcomplexPostExtractor, - "#options" : {"tags": True}, "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", "tags_character": "shani_(the_witcher)", diff --git a/test/results/imagefap.py b/test/results/imagefap.py index 99a22166..b4f3ab81 100644 --- a/test/results/imagefap.py +++ b/test/results/imagefap.py @@ -5,6 +5,7 @@ # published by the Free Software Foundation. from gallery_dl.extractor import imagefap +from gallery_dl import exception __tests__ = ( @@ -12,16 +13,14 @@ __tests__ = ( "#url" : "https://www.imagefap.com/gallery/7102714", "#category": ("", "imagefap", "gallery"), "#class" : imagefap.ImagefapGalleryExtractor, - "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", - "#sha1_metadata": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b", - "#sha1_content" : "694a0a57385980a6f90fbc296cadcd6c11ba2dab", + "#exception": exception.HttpError, }, { "#url" : "https://www.imagefap.com/gallery/7876223", "#category": ("", "imagefap", "gallery"), "#class" : imagefap.ImagefapGalleryExtractor, - "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", "#count" : 44, "categories" : [ @@ -90,7 +89,7 @@ __tests__ = ( "#url" : "https://www.imagefap.com/photo/1962981893", "#category": ("", "imagefap", "image"), "#class" : imagefap.ImagefapImageExtractor, - "#pattern" : r"https://cdnh?\.imagefap\.com/images/full/65/196/1962981893\.jpg", + "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/65/196/1962981893\.jpg", "date" : "21/08/2014", "gallery_id": 7876223, diff --git a/test/results/issuu.py b/test/results/issuu.py index 5e086d1e..4a90be13 100644 --- a/test/results/issuu.py +++ b/test/results/issuu.py @@ -25,7 +25,6 @@ __tests__ = ( "date" : "dt:2019-09-16 00:00:00", "description" : r"re:Motions, the brand new publication by I", "documentName" : "motions-1-2019", - "downloadable" : False, "pageCount" : 36, "publicationId": "d99ec95935f15091b040cb8060f05510", "title" : "Motions by Issuu - Issue 1", diff --git a/test/results/itaku.py b/test/results/itaku.py index ef3d3679..8a5b5066 100644 --- a/test/results/itaku.py +++ b/test/results/itaku.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://itaku.ee/profile/piku/gallery", "#category": ("", "itaku", "gallery"), "#class" : itaku.ItakuGalleryExtractor, - "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/[^/?#]+\.(jpg|png|gif)", + "#pattern" : r"https://itaku\.ee/api/media/gallery_imgs/[^/?#]+\.(jpg|png|gif)", "#range" : "1-10", "#count" : 10, }, @@ -21,8 +21,7 @@ __tests__ = ( "#url" : "https://itaku.ee/images/100471", "#category": ("", "itaku", "image"), "#class" : itaku.ItakuImageExtractor, - "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs/220504_oUNIAFT\.png", - "#count" : 1, + "#urls" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", "already_pinned" : None, "blacklisted" : { @@ -38,8 +37,8 @@ __tests__ = ( "filename" : "220504_oUNIAFT", "hotness_score" : float, "id" : 100471, - "image" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT.png", - "image_xl" : "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs/220504_oUNIAFT/lg.jpg", + "image" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", + "image_xl" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT/lg.jpg", "liked_by_you" : False, "maturity_rating" : "SFW", "num_comments" : int, @@ -47,7 +46,7 @@ __tests__ = ( "num_reshares" : int, "obj_tags" : 136446, "owner" : 16775, - "owner_avatar" : "https://d1wmr8tlk3viaj.cloudfront.net/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_avatar" : "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", "owner_displayname": "Piku", "owner_username" : "piku", "reshared_by_you" : False, @@ -55,13 +54,13 @@ __tests__ = ( "tags" : list, "tags_character" : ["hatsune_miku"], "tags_copyright" : ["vocaloid"], - "tags_general" : [ + "tags_general": [ + "female", + "green_eyes", "twintails", "green_hair", - "flag", "gloves", - "green_eyes", - "female", + "flag", "racing_miku", ], "title" : "Racing Miku 2022 Ver.", @@ -76,7 +75,7 @@ __tests__ = ( "#comment" : "video", "#category": ("", "itaku", "image"), "#class" : itaku.ItakuImageExtractor, - "#pattern" : r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids/sleepy_af_OY5GHWw\.mp4", + "#urls" : "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", }, ) diff --git a/test/results/itchio.py b/test/results/itchio.py index d2acfa61..f49bd69f 100644 --- a/test/results/itchio.py +++ b/test/results/itchio.py @@ -12,11 +12,11 @@ __tests__ = ( "#url" : "https://sirtartarus.itch.io/a-craft-of-mine", "#category": ("", "itchio", "game"), "#class" : itchio.ItchioGameExtractor, - "#pattern" : r"https://\w+\.ssl\.hwcdn\.net/upload2/game/1983311/7723751\?", - "#count" : 1, + "#pattern" : r"https://(dl.itch.zone|itchio-mirror.\w+.r2.cloudflarestorage.com)/upload2/game/1983311/\d+\?", + "#count" : 3, "extension": "", - "filename" : "7723751", + "filename" : r"re:\d+", "game" : { "id" : 1983311, "noun" : "game", diff --git a/test/results/khinsider.py b/test/results/khinsider.py index b680cab2..7013069f 100644 --- a/test/results/khinsider.py +++ b/test/results/khinsider.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", "#category": ("", "khinsider", "soundtrack"), "#class" : khinsider.KhinsiderSoundtrackExtractor, - "#pattern" : r"https?://vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", + "#pattern" : r"https?://(dl\.)?vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", "#count" : 1, "album" : { diff --git a/test/results/lesbianenergy.py b/test/results/lesbianenergy.py index fffe8a0c..650671f9 100644 --- a/test/results/lesbianenergy.py +++ b/test/results/lesbianenergy.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://lesbian.energy/@rerorero", "#category": ("misskey", "lesbian.energy", "user"), "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://lesbian.energy/files/\w+", + "#pattern" : r"https://(lesbian.energy/files/\w+|.+/media_attachments/files/.+)", "#range" : "1-50", "#count" : 50, }, diff --git a/test/results/luscious.py b/test/results/luscious.py index d5429612..5e7a1460 100644 --- a/test/results/luscious.py +++ b/test/results/luscious.py @@ -20,12 +20,12 @@ __tests__ = ( "__typename" : "Album", "audiences" : list, "content" : "Hentai", - "cover" : r"re:https://\w+.luscious.net/.+/277031/", + "cover" : r"re:https://storage\.bhs\.cloud\.ovh\.net/v1/.+/277031/", "created" : 1479625853, - "created_by" : "NTRshouldbeillegal", + "created_by" : "Hive Mind", "date" : "dt:2016-11-20 07:10:53", "description" : "Enjoy.", - "download_url" : r"re:/download/(r/)?824778/277031/", + "download_url" : "/download/r/25/277031/", "genres" : list, "id" : 277031, "is_manga" : True, @@ -34,7 +34,7 @@ __tests__ = ( "like_status" : "none", "modified" : int, "permissions" : list, - "rating" : float, + "rating" : None, "slug" : "okinami-no-koigokoro", "status" : None, "tags" : list, diff --git a/test/results/mangadex.py b/test/results/mangadex.py index c6a9e53a..17b2157c 100644 --- a/test/results/mangadex.py +++ b/test/results/mangadex.py @@ -97,7 +97,7 @@ __tests__ = ( "#url" : "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", "#category": ("", "mangadex", "manga"), "#class" : mangadex.MangadexMangaExtractor, - "#count" : 1, + "#count" : ">= 25", }, { diff --git a/test/results/misskeyio.py b/test/results/misskeyio.py index 3a28ae87..9d005483 100644 --- a/test/results/misskeyio.py +++ b/test/results/misskeyio.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://misskey.io/@lithla", "#category": ("misskey", "misskey.io", "user"), "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+", + "#pattern" : r"https://(media.misskeyusercontent.com/io|s\d+\.arkjp\.net/misskey)/[\w-]+\.\w+", "#range" : "1-50", "#count" : 50, }, diff --git a/test/results/myhentaigallery.py b/test/results/myhentaigallery.py index 6ddc2c2f..b7b5ac99 100644 --- a/test/results/myhentaigallery.py +++ b/test/results/myhentaigallery.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://myhentaigallery.com/gallery/thumbnails/16247", "#category": ("", "myhentaigallery", "gallery"), "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, - "#pattern" : r"https://images.myhentaicomics\.com/imagesgallery/images/[^/]+/original/\d+\.jpg", + "#pattern" : r"https://images\.myhentaicomics\.com/mhg/images/[^/]+/original/\d+\.jpg", "artist" : list, "count" : 11, diff --git a/test/results/nsfwalbum.py b/test/results/nsfwalbum.py index e89b8fb4..cabd3e10 100644 --- a/test/results/nsfwalbum.py +++ b/test/results/nsfwalbum.py @@ -13,8 +13,24 @@ __tests__ = ( "#category": ("", "nsfwalbum", "album"), "#class" : nsfwalbum.NsfwalbumAlbumExtractor, "#range" : "1-5", - "#sha1_url" : "b0481fc7fad5982da397b6359fbed8421b8ba284", - "#sha1_metadata": "e98f9b0d473c00000831618d0235863b1dd78294", + "#urls" : ( + "https://img70.imgspice.com/i/05457/mio2bu5xbrxe.jpg", + "https://img70.imgspice.com/i/05457/zgpxa8kr4h1d.jpg", + "https://img70.imgspice.com/i/05457/3379nxsm9lx8.jpg", + "https://img70.imgspice.com/i/05457/pncrkhspuoa3.jpg", + "https://img70.imgspice.com/i/05457/128b2odt216a.jpg", + ), + + "album_id" : 401611, + "extension": "jpg", + "filename" : str, + "height" : range(1365, 2048), + "id" : int, + "models" : [], + "num" : range(1, 5), + "studio" : "Met-Art", + "title" : "Met-Art - Katherine A - Difuza 25.05.2014 (134 photos)(4368 X 2912)", + "width" : range(1365, 2048), }, ) diff --git a/test/results/paheal.py b/test/results/paheal.py index 3ef0ec58..833f3f84 100644 --- a/test/results/paheal.py +++ b/test/results/paheal.py @@ -42,19 +42,19 @@ __tests__ = ( "#url" : "https://rule34.paheal.net/post/view/481609", "#category": ("shimmie2", "paheal", "post"), "#class" : paheal.PahealPostExtractor, - "#pattern" : r"https://tulip\.paheal\.net/_images/bbdc1c33410c2cdce7556c7990be26b7/481609%20-%20Azumanga_Daioh%20inanimate%20Osaka%20Vuvuzela\.jpg", + "#pattern" : r"https://tulip\.paheal\.net/_images/bbdc1c33410c2cdce7556c7990be26b7/481609%20-.+\.jpg", "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", "date" : "dt:2010-06-17 15:40:23", "extension": "jpg", "file_url" : r"re:https://tulip.paheal.net/_images/bbdc1c33410c", - "filename" : "481609 - Azumanga_Daioh inanimate Osaka Vuvuzela", + "filename" : "481609 - Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", "height" : 660, "id" : 481609, "md5" : "bbdc1c33410c2cdce7556c7990be26b7", "size" : 157389, "source" : "", - "tags" : "Azumanga_Daioh inanimate Osaka Vuvuzela", + "tags" : "Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", "uploader" : "CaptainButtface", "width" : 614, }, diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py index fea09746..b04be6f3 100644 --- a/test/results/pillowfort.py +++ b/test/results/pillowfort.py @@ -177,7 +177,7 @@ __tests__ = ( "#category": ("", "pillowfort", "user"), "#class" : pillowfort.PillowfortUserExtractor, "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", - "#count" : 6, + "#count" : range(10, 20), }, ) diff --git a/test/results/pornhub.py b/test/results/pornhub.py index e7aaf8da..e2aa9818 100644 --- a/test/results/pornhub.py +++ b/test/results/pornhub.py @@ -39,22 +39,32 @@ __tests__ = ( }, { - "#url" : "https://www.pornhub.com/gif/33643461", + "#url" : "https://www.pornhub.com/gif/43726891", "#category": ("", "pornhub", "gif"), "#class" : pornhub.PornhubGifExtractor, - "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/033/643/461/33643461a\.webm", + "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/043/726/891/43726891a\.webm", - "date" : "dt:2020-10-31 00:00:00", + "date" : "dt:2023-04-20 00:00:00", "extension": "webm", - "filename" : "33643461a", - "id" : "33643461", + "filename" : "43726891a", + "id" : "43726891", "tags" : [ - "big boobs", - "lana rhoades", + "sloppy deepthroat", + "perfect body", + "petite brunette", + "mouth fuck", + "big dick", + "natural big tits", + "deepthroat swallow", + "amateur couple", + "homemade", + "girls wanking boys", + "hardcore sex", + "babes 18 year", ], - "title" : "Big boobs", - "url" : str, - "user" : "Lana Rhoades", + "title" : "Intense sloppy blowjob of Danika Mori", + "url" : "https://el.phncdn.com/pics/gifs/043/726/891/43726891a.webm", + "user" : "Danika Mori", }, { diff --git a/test/results/pornpics.py b/test/results/pornpics.py index 91a10ca0..2bcdcfec 100644 --- a/test/results/pornpics.py +++ b/test/results/pornpics.py @@ -15,10 +15,10 @@ __tests__ = ( "#pattern" : r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", "categories": [ + "Outdoor", "MILF", "Amateur", "Sexy", - "Outdoor", ], "channel" : "FTV MILFs", "count" : 17, @@ -28,6 +28,9 @@ __tests__ = ( "slug" : "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", "tags" : [ "Amateur MILF", + "Nature", + "Amateur Outdoor", + "First Time", "Sexy MILF", ], "title" : "British beauty Danielle flashes hot breasts, ass and snatch in the forest", diff --git a/test/results/pururin.py b/test/results/pururin.py index e57c7fe7..971eb1d3 100644 --- a/test/results/pururin.py +++ b/test/results/pururin.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://pururin.to/gallery/38661/iowant-2", "#category": ("", "pururin", "gallery"), "#class" : pururin.PururinGalleryExtractor, - "#pattern" : r"https://i\.pururin\.to/38661/\d+\.jpg", + "#pattern" : r"https://i\.pururin\.[ct]o/38661/\d+\.jpg", "title" : r"re:I ?owant 2!!", "title_en" : r"re:I ?owant 2!!", diff --git a/test/results/reddit.py b/test/results/reddit.py index 8a4359cf..e5cd1c5e 100644 --- a/test/results/reddit.py +++ b/test/results/reddit.py @@ -189,7 +189,7 @@ __tests__ = ( "#comment" : "preview.redd.it (#4470)", "#category": ("", "reddit", "submission"), "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", + "#urls" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", }, { diff --git a/test/results/sexcom.py b/test/results/sexcom.py index 079bb5fa..2f212904 100644 --- a/test/results/sexcom.py +++ b/test/results/sexcom.py @@ -51,10 +51,9 @@ __tests__ = ( { "#url" : "https://www.sex.com/pin/55847384-very-nicely-animated/", - "#comment" : "pornhub embed", + "#comment" : "pornhub embed (404 gone)", "#category": ("", "sexcom", "pin"), "#class" : sexcom.SexcomPinExtractor, - "#pattern" : "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", }, { diff --git a/test/results/skeb.py b/test/results/skeb.py index f956986b..a8b546ad 100644 --- a/test/results/skeb.py +++ b/test/results/skeb.py @@ -58,7 +58,7 @@ __tests__ = ( "#url" : "https://skeb.jp/@kanade_cocotte", "#category": ("", "skeb", "user"), "#class" : skeb.SkebUserExtractor, - "#pattern" : r"https://skeb\.imgix\.net/uploads/origins/[\w-]+\?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&fm=webp&w=800&s=\w+", + "#pattern" : r"https://si\.imgix\.net/\w+/uploads/origins/[\w-]+", "#range" : "1-5", }, diff --git a/test/results/smugloli.py b/test/results/smugloli.py index 5d044123..f0176067 100644 --- a/test/results/smugloli.py +++ b/test/results/smugloli.py @@ -9,15 +9,15 @@ from gallery_dl.extractor import vichan __tests__ = ( { - "#url" : "https://smuglo.li/a/res/1154380.html", + "#url" : "https://smuglo.li/a/res/1187531.html", "#category": ("vichan", "smugloli", "thread"), "#class" : vichan.VichanThreadExtractor, "#pattern" : r"https://smug.+/a/src/\d+(-\d)?\.\w+", - "#count" : ">= 18", + "#count" : ">= 50", "board" : "a", - "thread": "1154380", - "title" : "Mob Psycho 100 Season 3", + "thread": "1187531", + "title" : "Buta no Liver wa Kanetsu Shiro", }, { diff --git a/test/results/tapas.py b/test/results/tapas.py index a3d79842..1278d9f8 100644 --- a/test/results/tapas.py +++ b/test/results/tapas.py @@ -12,7 +12,7 @@ __tests__ = ( "#url" : "https://tapas.io/series/just-leave-me-be", "#category": ("", "tapas", "series"), "#class" : tapas.TapasSeriesExtractor, - "#pattern" : r"https://\w+\.cloudfront\.net/pc/\w\w/[0-9a-f-]+\.jpg", + "#pattern" : r"https://us-a\.tapas\.io/pc/\w\w/[0-9a-f-]+\.jpg", "#count" : 132, }, @@ -64,7 +64,7 @@ __tests__ = ( "has_top_banner": True, "id" : 199931, "premium" : True, - "sale_type" : "PAID", + "sale_type" : "WAIT_OR_MUST_PAY", "subscribed" : bool, "thumbsup_cnt" : int, "title" : "Tomb Raider King", diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py index 22bc1412..21d13ee1 100644 --- a/test/results/toyhouse.py +++ b/test/results/toyhouse.py @@ -60,7 +60,7 @@ __tests__ = ( "http://aminoapps.com/p/92sf3z", "kroksoc (Color)", ], - "characters": ["❀Reiichi❀"], + "characters": ["Reiichi❀"], "date" : "dt:2021-07-03 20:02:02", "hash" : "bqhGcwcnU", "id" : "36817425", diff --git a/test/results/tumblr.py b/test/results/tumblr.py index 01d8de73..70e334b5 100644 --- a/test/results/tumblr.py +++ b/test/results/tumblr.py @@ -26,8 +26,8 @@ __tests__ = ( "posts" : "all", "external": True, }, - "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+_1280\.jpg|a\.tumblr\.com/tumblr_\w+)", - "#count" : 3, + "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+\.(jpg|png|gif|mp3|mp4)|v?a\.(media\.)?tumblr\.com/tumblr_\w+)", + "#count" : 27, }, { @@ -103,7 +103,7 @@ __tests__ = ( "date-max" : "2015-04-25T00:00:00", "date-min" : "2015-04-01T00:00:00", }, - "#count" : 316, + "#count" : 197, }, { diff --git a/test/results/turboimagehost.py b/test/results/turboimagehost.py index 3e2069fa..642d9321 100644 --- a/test/results/turboimagehost.py +++ b/test/results/turboimagehost.py @@ -14,7 +14,10 @@ __tests__ = ( "#class" : imagehosts.TurboimagehostImageExtractor, "#sha1_url" : "b94de43612318771ced924cb5085976f13b3b90e", "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", - "#sha1_content" : "f38b54b17cd7462e687b58d83f00fca88b1b105a", + "#sha1_content" : ( + "f38b54b17cd7462e687b58d83f00fca88b1b105a", + "0c8768055e4e20e7c7259608b67799171b691140", + ), }, ) diff --git a/test/results/twibooru.py b/test/results/twibooru.py index 5dd0191d..ff87deec 100644 --- a/test/results/twibooru.py +++ b/test/results/twibooru.py @@ -44,7 +44,7 @@ __tests__ = ( "tag_ids" : list, "tags" : list, "thumbnails_generated": True, - "updated_at" : "2022-11-27T00:34:50.483Z", + "updated_at" : "2023-07-24T03:18:48.153Z", "upvotes" : int, "view_url" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", "width" : 576, diff --git a/test/results/unsplash.py b/test/results/unsplash.py index 1568ed3a..e3413aff 100644 --- a/test/results/unsplash.py +++ b/test/results/unsplash.py @@ -9,69 +9,127 @@ from gallery_dl.extractor import unsplash __tests__ = ( { - "#url" : "https://unsplash.com/photos/lsoogGC_5dg", + "#url" : "https://unsplash.com/photos/red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", "#category": ("", "unsplash", "image"), "#class" : unsplash.UnsplashImageExtractor, - "#pattern" : r"https://images\.unsplash\.com/photo-1586348943529-beaae6c28db9\?ixid=\w+&ixlib=rb-4.0.3", + "#urls" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3", - "alt_description": r"re:silhouette of trees near body of water ", - "blur_hash" : "LZP4uQS4jboe%#o0WCa}2doJNaaz", - "? categories" : list, - "color" : "#f3c08c", - "created_at" : "2020-04-08T12:29:42Z", - "date" : "dt:2020-04-08 12:29:42", - "description" : "The Island", - "downloads" : int, - "exif" : { - "aperture" : "11", - "exposure_time": "30", - "focal_length" : "70.0", - "iso" : 200, - "make" : "Canon", - "model" : "Canon EOS 5D Mark IV", + "alt_description": "red wooden cross on gray concrete pathway between green trees during daytime", + "blur_hash" : "LIAwhq%e4TRjXAIBMyt89GRj%fj[", + "breadcrumbs": list, + "color" : "#0c2626", + "created_at" : "2020-10-04T15:13:59Z", + "date" : "dt:2020-10-04 15:13:59", + "description": None, + "downloads" : range(50000, 300000), + "exif" : { + "aperture" : "9", + "exposure_time": "1/125", + "focal_length" : "35.0", + "iso" : 800, + "make" : "SONY", + "model" : "ILCE-7M3", + "name" : "SONY, ILCE-7M3", }, - "extension" : "jpg", - "filename" : "photo-1586348943529-beaae6c28db9", - "height" : 6272, - "id" : "lsoogGC_5dg", - "liked_by_user" : False, - "likes" : int, - "location" : { - "city" : "Beaver Dam", - "country" : "United States", - "name" : "Beaver Dam, WI 53916, USA", + "extension" : "jpg", + "filename" : "photo-1601823984263-b87b59798b70", + "height" : 5371, + "id" : "kaoHI0iHJPM", + "liked_by_user": False, + "likes" : range(1000, 10000), + "links" : dict, + "location" : { + "city" : "箱根町", + "country" : "日本", + "name" : "Hakone, 神奈川県 日本", "position": { - "latitude" : 43.457769, - "longitude": -88.837329, + "latitude" : 35.232383, + "longitude": 139.106936, }, }, - "promoted_at" : "2020-04-08T15:12:03Z", - "sponsorship" : None, - "tags" : list, - "updated_at" : str, - "user" : { + "meta" : { + "index": True, + }, + "plus" : False, + "premium" : False, + "promoted_at": "2020-10-05T13:04:43Z", + "public_domain": False, + "slug" : "red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", + "sponsorship": None, + "subcategory": "image", + "tags" : [ + "japan", + "hakone", + "神奈川県 日本", + "torii", + "hakone shrine", + "sunrise", + "traditional", + "shrine", + "grey", + "wallpaper", + "arbour", + "garden", + "outdoors", + "gate", + ], + "tags_preview": list, + "topic_submissions": {}, + "topics" : [], + "updated_at" : "2023-11-24T08:17:36Z", + "urls": { + "full" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?crop=entropy&cs=srgb&fm=jpg&ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3&q=85", + "raw" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3", + "regular" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?crop=entropy&cs=tinysrgb&fit=max&fm=jpg&ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3&q=80&w=1080", + "small" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?crop=entropy&cs=tinysrgb&fit=max&fm=jpg&ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3&q=80&w=400", + "small_s3": "https://s3.us-west-2.amazonaws.com/images.unsplash.com/small/photo-1601823984263-b87b59798b70", + "thumb" : "https://images.unsplash.com/photo-1601823984263-b87b59798b70?crop=entropy&cs=tinysrgb&fit=max&fm=jpg&ixid=M3wxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNzAwODY2NDE4fA&ixlib=rb-4.0.3&q=80&w=200", + }, + "user": { "accepted_tos" : True, - "bio" : str, - "first_name" : "Dave", - "id" : "uMJXuywXLiU", - "instagram_username": "just_midwest_rock", - "last_name" : "Hoefler", - "location" : None, - "name" : "Dave Hoefler", - "portfolio_url" : None, - "total_collections" : int, - "total_likes" : int, - "total_photos" : int, + "bio" : "Professional photographer.\r\nBased in Japan.", + "first_name" : "Syuhei", + "for_hire" : True, + "id" : "F4HO358YSeo", + "instagram_username": "_______life_", + "last_name" : "Inoue", + "links": { + "followers": "https://api.unsplash.com/users/_______life_/followers", + "following": "https://api.unsplash.com/users/_______life_/following", + "html" : "https://unsplash.com/@_______life_", + "likes" : "https://api.unsplash.com/users/_______life_/likes", + "photos" : "https://api.unsplash.com/users/_______life_/photos", + "portfolio": "https://api.unsplash.com/users/_______life_/portfolio", + "self" : "https://api.unsplash.com/users/_______life_", + }, + "location" : "Yokohama, Japan", + "name" : "Syuhei Inoue", + "portfolio_url" : "https://syuheiinoue.life/", + "profile_image" : { + "large" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=128&h=128", + "medium": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=64&h=64", + "small" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=32&h=32", + }, + "social" : { + "instagram_username": "_______life_", + "paypal_email" : None, + "portfolio_url" : "https://syuheiinoue.life/", + "twitter_username" : None, + }, + "total_collections" : 2, + "total_likes" : 32, + "total_photos" : 86, + "total_promoted_photos": 24, "twitter_username" : None, - "updated_at" : str, - "username" : "davehoefler", + "updated_at" : "2023-11-24T19:15:32Z", + "username" : "_______life_" }, - "views" : int, - "width" : 4480, + "views": range(2000000, 10000000), + "width": 3581, }, { - "#url" : "https://unsplash.com/@davehoefler", + "#url" : "https://unsplash.com/@_______life_", "#category": ("", "unsplash", "user"), "#class" : unsplash.UnsplashUserExtractor, "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", @@ -80,12 +138,11 @@ __tests__ = ( }, { - "#url" : "https://unsplash.com/@davehoefler/likes", + "#url" : "https://unsplash.com/@_______life_/likes", "#category": ("", "unsplash", "favorite"), "#class" : unsplash.UnsplashFavoriteExtractor, "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, + "#count" : 31, }, { diff --git a/test/results/weibo.py b/test/results/weibo.py index f98b7fd9..639994c0 100644 --- a/test/results/weibo.py +++ b/test/results/weibo.py @@ -48,6 +48,7 @@ __tests__ = ( { "#url" : "https://weibo.com/1758989602?tabtype=home", + "#comment" : "'tabtype=home' is broken on website itself", "#category": ("", "weibo", "home"), "#class" : weibo.WeiboHomeExtractor, "#range" : "1-30", diff --git a/test/results/wikiart.py b/test/results/wikiart.py index fef4bad5..47eb3ec7 100644 --- a/test/results/wikiart.py +++ b/test/results/wikiart.py @@ -12,8 +12,50 @@ __tests__ = ( "#url" : "https://www.wikiart.org/en/thomas-cole", "#category": ("", "wikiart", "artist"), "#class" : wikiart.WikiartArtistExtractor, - "#sha1_url" : "6844f207a5063c499fc1d5651b03127bc4fe2f73", - "#sha1_metadata": "09230b5f504697119e267349bf92487e657a7384", + "#pattern" : "https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)", + "#count" : "> 100", + + "albums" : None, + "artist" : { + "OriginalArtistName": "Thomas Cole", + "activeYearsCompletion": None, + "activeYearsStart" : None, + "artistName" : "Thomas Cole", + "biography" : "Thomas Cole inspired the generation of American [url href=https://www.wikiart.org/en/paintings-by-genre/landscape]landscape[/url] painters that came to be known as the [url href=https://www.wikiart.org/en/artists-by-painting-school/hudson-river-school]Hudson River School[/url]. Born in Bolton-le-Moors, Lancashire, England, in 1801, at the age of seventeen he emigrated with his family to the United States, first working as a wood engraver in Philadelphia before going to Steubenville, Ohio, where his father had established a wallpaper manufacturing business. \n\nCole received rudimentary instruction from an itinerant artist, began painting portraits, genre scenes, and a few landscapes, and set out to seek his fortune through Ohio and Pennsylvania. He soon moved on to Philadelphia to pursue his art, inspired by paintings he saw at the Pennsylvania Academy of the Fine Arts. Moving to New York City in spring 1825, Cole made a trip up the Hudson River to the eastern Catskill Mountains. Based on his sketches there, he executed three landscapes that a city bookseller agreed to display in his window. Colonel [url href=https://www.wikiart.org/en/john-trumbull]John Trumbull[/url], already renowned as the painter of the American Revolution, saw Cole’s pictures and instantly purchased one, recommending the other two to his colleagues William Dunlap and [url href=https://www.wikiart.org/en/asher-brown-durand]Asher B. Durand[/url]. \n\nWhat Trumbull recognized in the work of the young painter was the perception of wildness inherent in American scenery that landscape artists had theretofore ignored. Trumbull brought Cole to the attention of various patrons, who began eagerly buying his work. Dunlap publicized the discovery of the new talent, and Cole was welcomed into New York’s cultural community, which included the poet and editor William Cullen Bryant and the author James Fenimore Cooper. Cole became one of the founding members of the National Academy of Design in 1825. Even as Cole expanded his travels and subjects to include scenes in the White Mountains of New Hampshire, he aspired to what he termed a “higher style of a landscape” that included narrative—some of the paintings in paired series—including biblical and literary subjects, such as Cooper’s popular [url href=https://www.wikiart.org/en/thomas-cole/scene-from-the-last-of-the-mohicans-by-james-fenimore-cooper-1827][i]Last of the Mohicans[/i][/url]. \n\nBy 1829, his success enabled him to take the Grand Tour of Europe and especially Italy, where he remained in 1831–32, visiting Florence, Rome, and Naples. Thereafter he painted many Italian subjects, like [url href=https://www.wikiart.org/en/thomas-cole/a-view-near-tivoli-morning-1832][i]View near Tivoli. Morning[/i][/url] (1832). The region around Rome, along with the classical myth, also inspired [url href=https://www.wikiart.org/en/thomas-cole/the-titan-s-goblet-1833][i]The Titan’s Goblet[/i][/url] (1833). Cole’s travels and the encouragement and patronage of the New York merchant Luman Reed culminated in his most ambitious historical landscape series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-course-of-empire,resultType:masonry][i]The Course of Empire[/i][/url] (1833–1836), five pictures dramatizing the rise and fall of an ancient classical state. \n\nCole also continued to paint, with ever-rising technical assurance, sublime American scenes such as the [url href=https://www.wikiart.org/en/thomas-cole/view-from-mount-holyoke-1836][i]View from Mount Holyoke[/i][/url] (1836), [url href=https://www.wikiart.org/en/thomas-cole/the-oxbow-the-connecticut-river-near-northampton-1836][i]The Oxbow[/i][/url] (1836), in which he included a portrait of himself painting the vista and [url href=https://www.wikiart.org/en/thomas-cole/view-on-the-catskill-early-autunm-1837][i]View on the Catskill—Early Autumn[/i][/url] (1836-1837), in which he pastorally interpreted the prospect of his beloved Catskill Mountains from the village of Catskill, where he had moved the year before and met his wife-to-be, Maria Bartow. \n\nThe artist’s marriage brought with it increasing religious piety manifested in the four-part series [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-voyage-of-life,resultType:masonry][i]The Voyage of Life[/i][/url] (1840). In it, a river journey represents the human passage through life to eternal reward. Cole painted and exhibited a replica of the series in Rome, where he returned in 1841–42, traveling south to Sicily. After his return, he lived and worked chiefly in Catskill, keeping up with art activity in New York primarily through Durand. He continued to produce American and foreign landscape subjects of incredible beauty, including the [url href=https://www.wikiart.org/en/thomas-cole/the-mountain-ford-1846][i]Mountain Ford[/i][/url] (1846). \n\nIn 1844, Cole welcomed into his Catskill studio the young [url href=https://www.wikiart.org/en/frederic-edwin-church]Frederic Church[/url], who studied with him until 1846 and went on to become the most renowned exponent of the generation that followed Cole. By 1846, Cole was at work on his largest and most ambitious series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-cross-and-the-world,resultType:masonry][i]The Cross and the World[/i][/url], but in February 1848 contracted pleurisy and died before completing it. \n\nThe paintings of Thomas Cole, like the writings of his contemporary Ralph Waldo Emerson, stand as monuments to the dreams and anxieties of the fledgling American nation during the mid-19th century; and they are also euphoric celebrations of its natural landscapes. Cole is considered the first artist to bring the eye of a European [url href=https://www.wikiart.org/en/artists-by-art-movement/romanticism]Romantic[/url] landscape painter to those environments, but also a figure whose idealism and religious sensibilities expressed a uniquely American spirit. In his works, we find the dramatic splendor of [url href=https://www.wikiart.org/en/caspar-david-friedrich]Caspar David Freidrich[/url] or [url href=https://www.wikiart.org/en/william-turner]J.M.W Turner[/url] transposed onto the Catskill and Adirondack Mountains. But whereas younger American painters such as [url href=https://www.wikiart.org/en/albert-bierstadt]Albert Bierstadt[/url] had come into direct contact with [url href=https://www.wikiart.org/en/artists-by-art-institution/kunstakademie-dusseldorf-dusseldorf-germany#!#resultType:masonry]The Düsseldorf School of painting[/url], and thus with the tradition in which they placed themselves, Cole was largely self-tutored, representing something of the archetypal American figure of the auto-didact.\n\nIn many ways, Cole's art epitomizes all contradictions of European settler culture in America. He was in love with the sublime wildness of the American landscape and sought to preserve it with his art, but his very presence in that landscape, and the development of his career, depended on the processes of urbanization and civilization which threatened it. From a modern perspective, Cole's Eurocentric gaze on seemingly empty wildernesses which had, in fact, been populated for centuries, also seems troubling; where Native Americans do appear in his work, as in [url href=https://www.wikiart.org/en/thomas-cole/falls-of-the-kaaterskill-1826][i]Falls of the Kaaterskill[/i][/url] (1826), it is as picturesque flecks rather than characterized participants in the scene.\n\nCole's legacy is evident in the work of future American artists who advanced the Hudson River style, including his student Frederic Edwin Church, Albert Bierstadt, Jasper Cropsey, Asher B. Durand, [url href=https://www.wikiart.org/en/george-inness]George Inness[/url], [url href=https://www.wikiart.org/en/john-frederick-kensett]John Kensett[/url], and [url href=https://www.wikiart.org/en/thomas-moran]Thomas Moran[/url]. Speaking more broadly, a whole sweep of 20th-century North-American art, from [url href=https://www.wikiart.org/en/artists-by-art-movement/precisionism]Precisionism[/url] to [url href=https://www.wikiart.org/en/artists-by-art-movement/environmental-art]Land Art[/url], might be seen to have inherited something of the grand scale and ambition of Cole's work. In this sense, his paintings capture not only the character of American culture during the mid-19th century but perhaps something more enduring about the open and expansive quality of that culture.", + "birthDay" : "/Date(-5330448000000)/", + "birthDayAsString" : "February 1, 1801", + "contentId" : 254330, + "deathDay" : "/Date(-3846441600000)/", + "deathDayAsString" : "February 11, 1848", + "dictonaries" : [ + 1368, + 11415, + 310, + ], + "gender" : "male", + "image" : "https://uploads8.wikiart.org/temp/19f6a140-59d2-4959-8d11-fd4ca582b7f2.jpg!Portrait.jpg", + "lastNameFirst" : "Cole Thomas", + "periodsOfWork" : "", + "relatedArtistsIds" : [], + "series" : "The Cross and the World\r\nThe Course of Empire\r\nThe Voyage of Life", + "story" : "http://en.wikipedia.org/wiki/Thomas_Cole", + "themes" : "", + "url" : "thomas-cole", + "wikipediaUrl" : "http://en.wikipedia.org/wiki/Thomas_Cole" + }, + "artistName" : "Thomas Cole", + "artistUrl" : "/en/thomas-cole", + "extension" : str, + "filename" : str, + "flags" : int, + "height" : int, + "id" : r"re:[0-9a-f]+", + "image" : str, + "map" : str, + "paintingUrl": r"re:/en/thomas-cole/.+", + "title" : str, + "width" : int, + "year" : str, }, { @@ -30,6 +72,9 @@ __tests__ = ( "#category": ("", "wikiart", "image"), "#class" : wikiart.WikiartImageExtractor, "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", + "#urls" : ( + "https://uploads5.wikiart.org/images/huang-shen/summer.jpg", + ), }, { @@ -37,6 +82,19 @@ __tests__ = ( "#category": ("", "wikiart", "artworks"), "#class" : wikiart.WikiartArtworksExtractor, "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", + "#urls" : ( + "https://uploads4.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement.jpg", + "https://uploads6.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement-1.jpg", + "https://uploads0.wikiart.org/images/hieronymus-bosch/tiptych-of-temptation-of-st-anthony-1506.jpg", + "https://uploads7.wikiart.org/images/matthias-grünewald/st-elizabeth-and-a-saint-woman-with-palm-1511.jpg", + "https://uploads2.wikiart.org/images/matthias-grünewald/st-lawrence-and-st-cyricus-1511.jpg", + "https://uploads0.wikiart.org/images/pieter-bruegel-the-elder/the-death-of-the-virgin.jpg", + "https://uploads4.wikiart.org/images/pieter-bruegel-the-elder/christ-and-the-woman-taken-in-adultery-1565-1.jpg", + "https://uploads6.wikiart.org/images/giovanni-battista-tiepolo/not_detected_241014.jpg", + "https://uploads4.wikiart.org/images/edgar-degas/interior-the-rape-1869.jpg", + "https://uploads3.wikiart.org/00265/images/john-singer-sargent/1396294310-dame-alice-ellen-terry-by-john-singer-sargent.jpg", + "https://uploads0.wikiart.org/00293/images/hryhorii-havrylenko/1954-18-5-32-5.jpg", + ), }, { diff --git a/test/results/wikifeet.py b/test/results/wikifeet.py index dade42e7..56e391c5 100644 --- a/test/results/wikifeet.py +++ b/test/results/wikifeet.py @@ -23,7 +23,7 @@ __tests__ = ( "pid" : int, "width" : int, "height" : int, - "shoesize" : "9 US", + "shoesize" : "10 US", "type" : "women", "tags" : list, },