diff --git a/docs/configuration.rst b/docs/configuration.rst index 00be43a7..b3344722 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3089,6 +3089,176 @@ Description Download video files. +extractor.steamgriddb.animated +------------------------------ +Type + ``bool`` +Default + ``true`` +Description + Include animated assets when downloading from a list of assets. + + +extractor.steamgriddb.epilepsy +------------------------------ +Type + ``bool`` +Default + ``true`` +Description + Include assets tagged with epilepsy when downloading from a list of assets. + + +extractor.steamgriddb.dimensions +-------------------------------- +Type + * ``string`` + * ``list`` of ``strings`` +Default + ``"all"`` +Examples + * ``"1024x512,512x512"`` + * ``["460x215", "920x430"]`` +Description + Only include assets that are in the specified dimensions. ``all`` can be + used to specify all dimensions. Valid values are: + + * Grids: ``460x215``, ``920x430``, ``600x900``, ``342x482``, ``660x930``, + ``512x512``, ``1024x1024`` + * Heroes: ``1920x620``, ``3840x1240``, ``1600x650`` + * Logos: N/A (will be ignored) + * Icons: ``8x8``, ``10x10``, ``14x14``, ``16x16``, ``20x20``, ``24x24``, + ``28x28``, ``32x32``, ``35x35``, ``40x40``, ``48x48``, ``54x54``, + ``56x56``, ``57x57``, ``60x60``, ``64x64``, ``72x72``, ``76x76``, + ``80x80``, ``90x90``, ``96x96``, ``100x100``, ``114x114``, ``120x120``, + ``128x128``, ``144x144``, ``150x150``, ``152x152``, ``160x160``, + ``180x180``, ``192x192``, ``194x194``, ``256x256``, ``310x310``, + ``512x512``, ``768x768``, ``1024x1024`` + + +extractor.steamgriddb.file-types +-------------------------------- +Type + * ``string`` + * ``list`` of ``strings`` +Default + ``"all"`` +Examples + * ``"png,jpeg"`` + * ``["jpeg", "webp"]`` +Description + Only include assets that are in the specified file types. ``all`` can be + used to specifiy all file types. Valid values are: + + * Grids: ``png``, ``jpeg``, ``jpg``, ``webp`` + * Heroes: ``png``, ``jpeg``, ``jpg``, ``webp`` + * Logos: ``png``, ``webp`` + * Icons: ``png``, ``ico`` + + +extractor.steamgriddb.download-fake-png +--------------------------------------- +Type + ``bool`` +Default + ``true`` +Description + Download fake PNGs alongside the real file. + + +extractor.steamgriddb.humor +--------------------------- +Type + ``bool`` +Default + ``true`` +Description + Include assets tagged with humor when downloading from a list of assets. + + +extractor.steamgriddb.languages +------------------------------- +Type + * ``string`` + * ``list`` of ``strings`` +Default + ``"all"`` +Examples + * ``"en,km"`` + * ``["fr", "it"]`` +Description + Only include assets that are in the specified languages. ``all`` can be + used to specifiy all languages. Valid values are `ISO 639-1 `__ + language codes. + + +extractor.steamgriddb.nsfw +-------------------------- +Type + ``bool`` +Default + ``true`` +Description + Include assets tagged with adult content when downloading from a list of assets. + + +extractor.steamgriddb.sort +-------------------------- +Type + ``string`` +Default + ``score_desc`` +Description + Set the chosen sorting method when downloading from a list of assets. Can be one of: + + * ``score_desc`` (Highest Score (Beta)) + * ``score_asc`` (Lowest Score (Beta)) + * ``score_old_desc`` (Highest Score (Old)) + * ``score_old_asc`` (Lowest Score (Old)) + * ``age_desc`` (Newest First) + * ``age_asc`` (Oldest First) + + +extractor.steamgriddb.static +---------------------------- +Type + ``bool`` +Default + ``true`` +Description + Include static assets when downloading from a list of assets. + + +extractor.steamgriddb.styles +---------------------------- +Type + * ``string`` + * ``list`` of ``strings`` +Default + ``all`` +Examples + * ``white,black`` + * ``["no_logo", "white_logo"]`` +Description + Only include assets that are in the specified styles. ``all`` can be used + to specify all styles. Valid values are: + + * Grids: ``alternate``, ``blurred``, ``no_logo``, ``material``, ``white_logo`` + * Heroes: ``alternate``, ``blurred``, ``material`` + * Logos: ``official``, ``white``, ``black``, ``custom`` + * Icons: ``official``, ``custom`` + + +extractor.steamgriddb.untagged +------------------------------ +Type + ``bool`` +Default + ``true`` +Description + Include untagged assets when downloading from a list of assets. + + extractor.[szurubooru].username & .token ---------------------------------------- Type diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 999bdba6..3a704cf4 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -817,6 +817,12 @@ Consider all listed sites to potentially be NSFW. Presentations + + SteamGridDB + https://www.steamgriddb.com + Individual Assets, Grids, Heroes, Icons, Logos + + SubscribeStar https://www.subscribestar.com/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 26ce2093..13d7b38b 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -146,6 +146,7 @@ modules = [ "smugmug", "soundgasm", "speakerdeck", + "steamgriddb", "subscribestar", "szurubooru", "tapas", diff --git a/gallery_dl/extractor/steamgriddb.py b/gallery_dl/extractor/steamgriddb.py new file mode 100644 index 00000000..9d46fd6b --- /dev/null +++ b/gallery_dl/extractor/steamgriddb.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.steamgriddb.com""" + +from .common import Extractor, Message +from .. import text, exception + + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?steamgriddb\.com" +LANGUAGE_CODES = ( + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", + "ba", "be", "bg", "bh", "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", + "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", "de", "dv", "dz", "ee", + "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", + "ht", "hu", "hy", "hz", "ia", "id", "ie", "ig", "ii", "ik", "io", "is", + "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", "kk", "kl", "km", "kn", + "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", + "mt", "my", "na", "nb", "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", + "ny", "oc", "oj", "om", "or", "os", "pa", "pi", "pl", "ps", "pt", "qu", + "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", + "te", "tg", "th", "ti", "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", + "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", "wa", "wo", "xh", "yi", + "yo", "za", "zh", "zu", +) +FILE_EXT_TO_MIME = { + "png": "image/png", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "webp": "image/webp", + "ico": "image/vnd.microsoft.icon", + "all": "all", +} + + +class SteamgriddbExtractor(Extractor): + """Base class for SteamGridDB""" + category = "steamgriddb" + directory_fmt = ("{category}", "{subcategory}", "{game[id]}") + filename_fmt = "{game[id]}_{id}_{num:>02}.{extension}" + archive_fmt = "{filename}" + root = "https://www.steamgriddb.com" + + def _init(self): + self.cookies_update({ + "userprefs": "%7B%22adult%22%3Afalse%7D", + }) + + def items(self): + download_fake_png = self.config("download-fake-png", True) + + for asset in self.assets(): + if download_fake_png and asset.get("fake_png"): + urls = (asset["url"], asset["fake_png"]) + else: + urls = (asset["url"],) + + asset["count"] = len(urls) + yield Message.Directory, asset + for asset["num"], url in enumerate(urls, 1): + yield Message.Url, url, text.nameext_from_url(url, asset) + + def _call(self, endpoint, **kwargs): + data = self.request(self.root + endpoint, **kwargs).json() + if not data["success"]: + raise exception.StopExtraction(data["error"]) + return data["data"] + + +class SteamgriddbAssetsExtractor(SteamgriddbExtractor): + """Base class for extracting a list of assets""" + + def __init__(self, match): + SteamgriddbExtractor.__init__(self, match) + list_type = match.group(1) + id = int(match.group(2)) + self.game_id = id if list_type == "game" else None + self.collection_id = id if list_type == "collection" else None + self.page = int(match.group(3) or 1) + + def assets(self): + limit = 48 + page = min(self.page - 1, 0) + + sort = self.config("sort", "score_desc") + if sort not in ("score_desc", "score_asc", "score_old_desc", + "score_old_asc", "age_desc", "age_asc"): + raise exception.StopExtractor("Invalid sort '%s'", sort) + + json = { + "static" : self.config("static", True), + "animated": self.config("animated", True), + "humor" : self.config("humor", True), + "nsfw" : self.config("nsfw", True), + "epilepsy": self.config("epilepsy", True), + "untagged": self.config("untagged", True), + + "asset_type": self.asset_type, + "limit": limit, + "order": sort, + } + if self.valid_dimensions: + json["dimensions"] = self.config_list( + "dimensions", "dimension", self.valid_dimensions) + json["styles"] = self.config_list("styles", "style", self.valid_styles) + json["languages"] = self.config_list( + "languages", "language", LANGUAGE_CODES) + file_types = self.config_list( + "file-types", "file type", self.valid_file_types) + json["mime"] = [FILE_EXT_TO_MIME[i] for i in file_types] + + if self.game_id: + json["game_id"] = [self.game_id] + else: + json["collection_id"] = self.collection_id + + while True: + json["page"] = page + + data = self._call( + "/api/public/search/assets", method="POST", json=json) + for asset in data["assets"]: + if not asset.get("game"): + asset["game"] = data["game"] + yield asset + + if data["total"] <= limit * page: + break + page += 1 + + def config_list(self, key, type_name, valid_values): + value = self.config(key) + if isinstance(value, str): + value = value.split(",") + + if value is None or "all" in value: + return ["all"] + + for i in value: + if i not in valid_values: + raise exception.StopExtraction("Invalid %s '%s'", type_name, i) + + return value + + +class SteamgriddbAssetExtractor(SteamgriddbExtractor): + """Extractor for a single asset""" + subcategory = "asset" + pattern = BASE_PATTERN + r"/(grid|hero|logo|icon)/(\d+)" + example = "https://www.steamgriddb.com/grid/1234" + + def __init__(self, match): + SteamgriddbExtractor.__init__(self, match) + self.asset_type = match.group(1) + self.asset_id = match.group(2) + + def assets(self): + endpoint = "/api/public/asset/" + self.asset_type + "/" + self.asset_id + asset = self._call(endpoint)["asset"] + return (asset,) + + +class SteamgriddbGridsExtractor(SteamgriddbAssetsExtractor): + subcategory = "grids" + asset_type = "grid" + pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/grids(?:/(\d+))?" + example = "https://www.steamgriddb.com/game/1234/grids" + valid_dimensions = ("460x215", "920x430", "600x900", "342x482", "660x930", + "512x512", "1024x1024") + valid_styles = ("alternate", "blurred", "no_logo", "material", + "white_logo") + valid_file_types = ("png", "jpeg", "jpg", "webp") + + +class SteamgriddbHeroesExtractor(SteamgriddbAssetsExtractor): + subcategory = "heroes" + asset_type = "hero" + pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/heroes(?:/(\d+))?" + example = "https://www.steamgriddb.com/game/1234/heroes" + valid_dimensions = ("1920x620", "3840x1240", "1600x650") + valid_styles = ("alternate", "blurred", "material") + valid_file_types = ("png", "jpeg", "jpg", "webp") + + +class SteamgriddbLogosExtractor(SteamgriddbAssetsExtractor): + subcategory = "logos" + asset_type = "logo" + pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/logos(?:/(\d+))?" + example = "https://www.steamgriddb.com/game/1234/logos" + valid_dimensions = None + valid_styles = ("official", "white", "black", "custom") + valid_file_types = ("png", "webp") + + +class SteamgriddbIconsExtractor(SteamgriddbAssetsExtractor): + subcategory = "icons" + asset_type = "icon" + pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/icons(?:/(\d+))?" + example = "https://www.steamgriddb.com/game/1234/icons" + valid_dimensions = ["{0}x{0}".format(i) for i in (8, 10, 14, 16, 20, 24, + 28, 32, 35, 40, 48, 54, 56, 57, 60, 64, 72, 76, 80, 90, + 96, 100, 114, 120, 128, 144, 150, 152, 160, 180, 192, + 194, 256, 310, 512, 768, 1024)] + valid_styles = ("official", "custom") + valid_file_types = ("png", "ico") diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index d29001c7..d3107b47 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -121,6 +121,7 @@ CATEGORY_MAP = { "slideshare" : "SlideShare", "smugmug" : "SmugMug", "speakerdeck" : "Speaker Deck", + "steamgriddb" : "SteamGridDB", "subscribestar" : "SubscribeStar", "tbib" : "The Big ImageBoard", "tcbscans" : "TCB Scans", @@ -267,6 +268,9 @@ SUBCATEGORY_MAP = { "smugmug": { "path": "Images from Users and Folders", }, + "steamgriddb": { + "asset": "Individual Assets", + }, "tumblr": { "day": "Days", }, diff --git a/test/results/steamgriddb.py b/test/results/steamgriddb.py new file mode 100644 index 00000000..06c1c22b --- /dev/null +++ b/test/results/steamgriddb.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import steamgriddb + + +__tests__ = ( +{ + "#url" : "https://www.steamgriddb.com/grid/368023", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, + "#urls" : ("https://cdn2.steamgriddb.com/grid/" + "82fee171d62c044898d99ba0fddeb203.png"), + "#count" : 1, + "#sha1_content": "0bffaccae6f35f9fab529684a5b158d1cec4186b", + + "game": { + "id" : 5259324, + "name": "Helltaker", + }, +}, + +{ + "#url" : "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, + "#count" : 2, + "#sha1_url" : "4ff9158c008a1f01921d7553bcabf5e6204cdc79", + "#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2", + + "game": { + "id" : 5247997, + "name": "OMORI", + }, +}, + +{ + "#url" : "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, + "#options" : {"download-fake-png": False}, + "#count" : 1, + "#sha1_url" : "f6819c593ff65f15864796fb89581f05d21adddb", + "#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd", + + "game": { + "id" : 5247997, + "name": "OMORI", + }, +}, + +{ + "#url" : "https://www.steamgriddb.com/hero/61104", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/logo/9610", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/icon/173", + "#category": ("", "steamgriddb", "asset"), + "#class" : steamgriddb.SteamgriddbAssetExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class" : steamgriddb.SteamgriddbGridsExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class" : steamgriddb.SteamgriddbGridsExtractor, + "#options" : {"humor": False, "epilepsy": False, "untagged": False}, + "#range" : "1-33", + "#count" : 33, +}, + +{ + "#url" : "https://www.steamgriddb.com/game/5331605/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class" : steamgriddb.SteamgriddbHeroesExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/game/5255394/logos", + "#category": ("", "steamgriddb", "logos"), + "#class" : steamgriddb.SteamgriddbLogosExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/game/5279790/icons", + "#category": ("", "steamgriddb", "icons"), + "#class" : steamgriddb.SteamgriddbIconsExtractor, +}, + +{ + "#url" : "https://www.steamgriddb.com/collection/332/grids", + "#category": ("", "steamgriddb", "grids"), + "#class" : steamgriddb.SteamgriddbGridsExtractor, + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://www.steamgriddb.com/collection/332/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class" : steamgriddb.SteamgriddbHeroesExtractor, + "#options" : {"animated": False}, + "#count" : 0, +}, + +)