From 2007cb2f59f0d509c4fe08a8e7c131b3067f3a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 19 Jan 2024 03:15:30 +0100 Subject: [PATCH] [tests] check extractor category values --- gallery_dl/extractor/mangadex.py | 3 ++- test/results/blogger.py | 2 +- test/results/mariowiki.py | 2 +- test/results/raddle.py | 24 ++++++++++++------------ test/results/wikimediacommons.py | 3 ++- test/test_extractor.py | 23 +++++++++++++++++------ 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index d287d5cf..bca7e4db 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -158,8 +158,9 @@ class MangadexListExtractor(MangadexExtractor): def __init__(self, match): MangadexExtractor.__init__(self, match) - if match.group(2) != "feed": + if match.group(2) == "feed": self.subcategory = "list-feed" + else: self.items = self._items_titles def chapters(self): diff --git a/test/results/blogger.py b/test/results/blogger.py index aeb82f76..eef96459 100644 --- a/test/results/blogger.py +++ b/test/results/blogger.py @@ -24,7 +24,7 @@ __tests__ = ( { "#url" : "blogger:http://www.julianbunker.com/search?q=400mm", - "#category": ("blogger", "1www.julianbunker.com", "search"), + "#category": ("blogger", "www.julianbunker.com", "search"), "#class" : blogger.BloggerSearchExtractor, }, diff --git a/test/results/mariowiki.py b/test/results/mariowiki.py index ebb8d6e6..72c4cd52 100644 --- a/test/results/mariowiki.py +++ b/test/results/mariowiki.py @@ -10,7 +10,7 @@ from gallery_dl.extractor import wikimedia __tests__ = ( { "#url" : "https://www.mariowiki.com/Rabbit", - "#category": ("wikimedia", "wikibooks", "article"), + "#category": ("wikimedia", "mariowiki", "article"), "#class" : wikimedia.WikimediaArticleExtractor, "#pattern" : r"https://mario\.wiki\.gallery/images/.+", "#count" : range(20, 50), diff --git a/test/results/raddle.py b/test/results/raddle.py index 0c9de429..24710e94 100644 --- a/test/results/raddle.py +++ b/test/results/raddle.py @@ -10,7 +10,7 @@ from gallery_dl.extractor import postmill __tests__ = ( { "#url" : "https://raddle.me/", - "#category": ("postmill", "raddle.me", "home"), + "#category": ("postmill", "raddle", "home"), "#class" : postmill.PostmillHomeExtractor, "#range" : "1-25", "#count" : 25, @@ -18,7 +18,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/traa", - "#category": ("postmill", "raddle.me", "forum"), + "#category": ("postmill", "raddle", "forum"), "#class" : postmill.PostmillForumExtractor, "#count" : 1, "#pattern" : r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$", @@ -26,7 +26,7 @@ __tests__ = ( { "#url" : "https://raddle.me/user/Sam_the_enby/submissions", - "#category": ("postmill", "raddle.me", "usersubmissions"), + "#category": ("postmill", "raddle", "usersubmissions"), "#class" : postmill.PostmillUserSubmissionsExtractor, "#range" : "1-25", "#count" : 25, @@ -34,13 +34,13 @@ __tests__ = ( { "#url" : "https://raddle.me/tag/Trans", - "#category": ("postmill", "raddle.me", "tag"), + "#category": ("postmill", "raddle", "tag"), "#class" : postmill.PostmillTagExtractor, }, { "#url" : "https://raddle.me/search?q=tw", - "#category": ("postmill", "raddle.me", "search"), + "#category": ("postmill", "raddle", "search"), "#class" : postmill.PostmillSearchExtractor, "#range" : "1-50", "#count" : 50, @@ -48,7 +48,7 @@ __tests__ = ( { "#url" : "https://raddle.me/160845", - "#category": ("postmill", "raddle.me", "shorturl"), + "#category": ("postmill", "raddle", "shorturl"), "#class" : postmill.PostmillShortURLExtractor, "#pattern" : r"^https://raddle\.me/f/egg_irl/160845/egg_irl$", }, @@ -56,7 +56,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/NonBinary/179017/scattered-thoughts-would-appreciate-advice-immensely-tw", "#comment" : "Text post", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "#sha1_url" : "99277f815820810d9d7e219d455f818601858378", "#sha1_content": "7a1159e1e45f2ce8e2c8b5959f6d66b042776f3b", @@ -66,7 +66,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/egg_irl/160845", "#comment" : "Image post", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "#sha1_url" : "48663f767ea258fcd545ab5aa0e734f98f434388", "#sha1_content": "431e938082c2b59c44888a83cfc711cd1f0e910a", @@ -76,7 +76,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/trans/177042/tw-vent-nsfw-suicide-i-lost-no-nut-november-tw-trauma", "#comment" : "Image + text post (with text enabled)", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "#options" : {"save-link-post-body": True}, "#pattern" : r"^(text:[\s\S]+|https://raddle\.me/submission_images/[0-9a-f]+\.png)$", @@ -86,7 +86,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/videos/179541/raisins-and-sprite", "#comment" : "Link post", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "#urls" : "https://m.youtube.com/watch?v=RFJCA5zcZxI", "#count" : 1, @@ -95,7 +95,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/Anime/150698/neo-tokyo-1987-link-to-the-english-dub-version-last-link", "#comment" : "Link + text post (with text disabled)", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "#pattern" : r"^https://fantasyanime\.com/anime/neo-tokyo-dub$", "#count" : 1, @@ -104,7 +104,7 @@ __tests__ = ( { "#url" : "https://raddle.me/f/egg_irl/166855/4th-wall-breaking-please-let-this-be-a-flair-egg-irl", "#comment" : "Post with multiple flairs", - "#category": ("postmill", "raddle.me", "post"), + "#category": ("postmill", "raddle", "post"), "#class" : postmill.PostmillPostExtractor, "flair" : ["Gender non-specific", "4th wall breaking"], }, diff --git a/test/results/wikimediacommons.py b/test/results/wikimediacommons.py index a16d069a..b61a9061 100644 --- a/test/results/wikimediacommons.py +++ b/test/results/wikimediacommons.py @@ -10,8 +10,9 @@ from gallery_dl.extractor import wikimedia __tests__ = ( { "#url" : "https://commons.wikimedia.org/wiki/File:Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_(24762757525).jpg", - "#category": ("wikimedia", "wikimediacommons", "article"), + "#category": ("wikimedia", "wikimediacommons", "file"), "#class" : wikimedia.WikimediaArticleExtractor, + "#urls" : "https://upload.wikimedia.org/wikipedia/commons/f/fa/Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_%2824762757525%29.jpg", }, { diff --git a/test/test_extractor.py b/test/test_extractor.py index d2dd643c..3b590ff4 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -24,6 +24,11 @@ from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 _list_classes = extractor._list_classes +try: + from test import results +except ImportError: + results = None + class FakeExtractor(Extractor): category = "fake" @@ -92,17 +97,23 @@ class TestExtractorModule(unittest.TestCase): with self.assertRaises(TypeError): FakeExtractor.from_url(invalid) + @unittest.skipIf(not results, "no test data") + def test_categories(self): + for result in results.all(): + url = result["#url"] + extr = result["#class"].from_url(url) + base, cat, sub = result["#category"] + self.assertEqual(extr.category, cat, url) + self.assertEqual(extr.subcategory, sub, url) + self.assertEqual(extr.basecategory, base, url) + + @unittest.skipIf(not results, "no test data") def test_unique_pattern_matches(self): - try: - import test.results - except ImportError: - raise unittest.SkipTest("no test data") - # collect testcase URLs test_urls = [] append = test_urls.append - for result in test.results.all(): + for result in results.all(): append((result["#url"], result["#class"])) # iterate over all testcase URLs