diff --git a/docs/configuration.rst b/docs/configuration.rst index 6772c9f4..1b61a4c9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1527,12 +1527,33 @@ Description extractor.deviantart.metadata ----------------------------- Type - ``bool`` + * ``bool`` + * ``string`` + * ``list`` of ``strings`` Default ``false`` +Example + * ``"stats,submission"`` + * ``["camera", "stats", "submission"]`` Description - Request extended metadata for deviation objects to additionally provide - ``description``, ``tags``, ``license`` and ``is_watching`` fields. + Extract additional metadata for deviation objects. + + Provides + ``description``, ``tags``, ``license``, and ``is_watching`` + fields when enabled. + + It is possible to request extended metadata by specifying a list of + + * ``camera`` : EXIF information (if available) + * ``stats`` : deviation statistics + * ``submission`` : submission information + * ``collection`` : favourited folder information (requires a `refresh token `__) + * ``gallery`` : gallery folder information (requires a `refresh token `__) + + Set this option to ``"all"`` to request all extended metadata categories. + + See `/deviation/metadata `__ + for official documentation. extractor.deviantart.original diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 0cf4f885..5a0b2c87 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1079,6 +1079,7 @@ class DeviantartOAuthAPI(): self.headers = {"dA-minor-version": "20200519"} self._warn_429 = True + self.limit = None self.delay = extractor.config("wait-min", 0) self.delay_min = max(2, self.delay) @@ -1086,9 +1087,8 @@ class DeviantartOAuthAPI(): if not isinstance(self.mature, str): self.mature = "true" if self.mature else "false" - self.folders = extractor.config("folders", False) - self.metadata = extractor.extra or extractor.config("metadata", False) self.strategy = extractor.config("pagination") + self.folders = extractor.config("folders", False) self.public = extractor.config("public", True) client_id = extractor.config("client-id") @@ -1106,6 +1106,41 @@ class DeviantartOAuthAPI(): token = None self.refresh_token_key = token + metadata = extractor.config("metadata", False) + if not metadata: + metadata = bool(extractor.extra) + if metadata: + self.metadata = True + + if isinstance(metadata, str): + if metadata == "all": + metadata = ("submission", "camera", "stats", + "collection", "gallery") + else: + metadata = metadata.replace(" ", "").split(",") + elif not isinstance(metadata, (list, tuple)): + metadata = () + + self._metadata_params = {"mature_content": self.mature} + self._metadata_public = None + if metadata: + # extended metadata + self.limit = 10 + for param in metadata: + self._metadata_params["ext_" + param] = "1" + if "ext_collection" in self._metadata_params or \ + "ext_gallery" in self._metadata_params: + if token: + self._metadata_public = False + else: + self.log.error("'collection' and 'gallery' metadata " + "require a refresh token") + else: + # base metadata + self.limit = 50 + else: + self.metadata = False + self.log.debug( "Using %s API credentials (client-id %s)", "default" if self.client_id == self.CLIENT_ID else "custom", @@ -1115,14 +1150,14 @@ class DeviantartOAuthAPI(): def browse_deviantsyouwatch(self, offset=0): """Yield deviations from users you watch""" endpoint = "/browse/deviantsyouwatch" - params = {"limit": "50", "offset": offset, + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False) def browse_posts_deviantsyouwatch(self, offset=0): """Yield posts from users you watch""" endpoint = "/browse/posts/deviantsyouwatch" - params = {"limit": "50", "offset": offset, + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False, unpack=True) @@ -1131,7 +1166,7 @@ class DeviantartOAuthAPI(): endpoint = "/browse/newest" params = { "q" : query, - "limit" : 50 if self.metadata else 120, + "limit" : 120, "offset" : offset, "mature_content": self.mature, } @@ -1142,7 +1177,7 @@ class DeviantartOAuthAPI(): endpoint = "/browse/popular" params = { "q" : query, - "limit" : 50 if self.metadata else 120, + "limit" : 120, "timerange" : timerange, "offset" : offset, "mature_content": self.mature, @@ -1249,8 +1284,11 @@ class DeviantartOAuthAPI(): "deviationids[{}]={}".format(num, deviation["deviationid"]) for num, deviation in enumerate(deviations) ) - params = {"mature_content": self.mature} - return self._call(endpoint, params=params)["metadata"] + return self._call( + endpoint, + params=self._metadata_params.copy(), + public=self._metadata_public, + )["metadata"] def gallery(self, username, folder_id, offset=0, extend=True, public=None): """Yield all Deviation-objects contained in a gallery folder""" @@ -1411,6 +1449,8 @@ class DeviantartOAuthAPI(): warn = True if public is None: public = self.public + if self.limit and params["limit"] > self.limit: + params["limit"] = self.limit while True: data = self._call(endpoint, params=params, public=public) diff --git a/gallery_dl/version.py b/gallery_dl/version.py index e89ab9ca..b48cd1e5 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.26.8" +__version__ = "1.26.9-dev" diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 60182890..5fd17761 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -642,6 +642,37 @@ __tests__ = ( "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", }, +{ + "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class" : deviantart.DeviantartDeviationExtractor, + "#options" : {"metadata": "submission,camera,stats"}, + + "can_post_comment": False, + "description" : str, + "is_watching" : False, + "license" : "No License", + "stats": { + "comments" : int, + "downloads" : int, + "downloads_today": int, + "favourites" : int, + "views" : int, + "views_today" : int, + }, + "submission": { + "category" : "traditional/drawings/other", + "creation_time" : "2004-08-25T02:44:08-0700", + "file_size" : "133 KB", + "resolution" : "710x510", + "submitted_with": { + "app": "Unknown App", + "url": "" + }, + }, + "tags": [], +}, + { "#url" : "https://www.deviantart.com/zzz/art/zzz-1234567890", "#category": ("", "deviantart", "deviation"),