[deviantart] extend 'metadata' option (#5175)

alloe fetching extended metadata in addition to the usual
'description', 'tags', etc by setting 'metadata' to a list of
'camera', 'stats', 'submission', 'collection', and 'gallery'

for example "metadata": "stats,submission"

docs/configuration.rst

@@ -1527,12 +1527,33 @@ Description
 extractor.deviantart.metadata
 -----------------------------
 Type
-    ``bool``
+    * ``bool``
+    * ``string``
+    * ``list`` of ``strings``
 Default
     ``false``
+Example
+    * ``"stats,submission"``
+    * ``["camera", "stats", "submission"]``
 Description
-    Request extended metadata for deviation objects to additionally provide
-    ``description``, ``tags``, ``license`` and ``is_watching`` fields.
+    Extract additional metadata for deviation objects.
+
+    Provides
+    ``description``, ``tags``, ``license``, and ``is_watching``
+    fields when enabled.
+
+    It is possible to request extended metadata by specifying a list of
+
+    * ``camera``     : EXIF information (if available)
+    * ``stats``      : deviation statistics
+    * ``submission`` : submission information
+    * ``collection`` : favourited folder information (requires a `refresh token <extractor.deviantart.refresh-token_>`__)
+    * ``gallery``    : gallery folder information (requires a `refresh token <extractor.deviantart.refresh-token_>`__)
+
+    Set this option to ``"all"`` to request all extended metadata categories.
+
+    See `/deviation/metadata <https://www.deviantart.com/developers/http/v1/20210526/deviation_metadata/7824fc14d6fba6acbacca1cf38c24158>`__
+    for official documentation.
 
 
 extractor.deviantart.original

gallery_dl/extractor/deviantart.py

@@ -1079,6 +1079,7 @@ class DeviantartOAuthAPI():
         self.headers = {"dA-minor-version": "20200519"}
         self._warn_429 = True
 
+        self.limit = None
         self.delay = extractor.config("wait-min", 0)
         self.delay_min = max(2, self.delay)
 
@@ -1086,9 +1087,8 @@ class DeviantartOAuthAPI():
         if not isinstance(self.mature, str):
             self.mature = "true" if self.mature else "false"
 
-        self.folders = extractor.config("folders", False)
-        self.metadata = extractor.extra or extractor.config("metadata", False)
         self.strategy = extractor.config("pagination")
+        self.folders = extractor.config("folders", False)
         self.public = extractor.config("public", True)
 
         client_id = extractor.config("client-id")
@@ -1106,6 +1106,41 @@ class DeviantartOAuthAPI():
                 token = None
         self.refresh_token_key = token
 
+        metadata = extractor.config("metadata", False)
+        if not metadata:
+            metadata = bool(extractor.extra)
+        if metadata:
+            self.metadata = True
+
+            if isinstance(metadata, str):
+                if metadata == "all":
+                    metadata = ("submission", "camera", "stats",
+                                "collection", "gallery")
+                else:
+                    metadata = metadata.replace(" ", "").split(",")
+            elif not isinstance(metadata, (list, tuple)):
+                metadata = ()
+
+            self._metadata_params = {"mature_content": self.mature}
+            self._metadata_public = None
+            if metadata:
+                # extended metadata
+                self.limit = 10
+                for param in metadata:
+                    self._metadata_params["ext_" + param] = "1"
+                if "ext_collection" in self._metadata_params or \
+                        "ext_gallery" in self._metadata_params:
+                    if token:
+                        self._metadata_public = False
+                    else:
+                        self.log.error("'collection' and 'gallery' metadata "
+                                       "require a refresh token")
+            else:
+                # base metadata
+                self.limit = 50
+        else:
+            self.metadata = False
+
         self.log.debug(
             "Using %s API credentials (client-id %s)",
             "default" if self.client_id == self.CLIENT_ID else "custom",
@@ -1115,14 +1150,14 @@ class DeviantartOAuthAPI():
     def browse_deviantsyouwatch(self, offset=0):
         """Yield deviations from users you watch"""
         endpoint = "/browse/deviantsyouwatch"
-        params = {"limit": "50", "offset": offset,
+        params = {"limit": 50, "offset": offset,
                   "mature_content": self.mature}
         return self._pagination(endpoint, params, public=False)
 
     def browse_posts_deviantsyouwatch(self, offset=0):
         """Yield posts from users you watch"""
         endpoint = "/browse/posts/deviantsyouwatch"
-        params = {"limit": "50", "offset": offset,
+        params = {"limit": 50, "offset": offset,
                   "mature_content": self.mature}
         return self._pagination(endpoint, params, public=False, unpack=True)
 
@@ -1131,7 +1166,7 @@ class DeviantartOAuthAPI():
         endpoint = "/browse/newest"
         params = {
             "q"             : query,
-            "limit"         : 50 if self.metadata else 120,
+            "limit"         : 120,
             "offset"        : offset,
             "mature_content": self.mature,
         }
@@ -1142,7 +1177,7 @@ class DeviantartOAuthAPI():
         endpoint = "/browse/popular"
         params = {
             "q"             : query,
-            "limit"         : 50 if self.metadata else 120,
+            "limit"         : 120,
             "timerange"     : timerange,
             "offset"        : offset,
             "mature_content": self.mature,
@@ -1249,8 +1284,11 @@ class DeviantartOAuthAPI():
             "deviationids[{}]={}".format(num, deviation["deviationid"])
             for num, deviation in enumerate(deviations)
         )
-        params = {"mature_content": self.mature}
-        return self._call(endpoint, params=params)["metadata"]
+        return self._call(
+            endpoint,
+            params=self._metadata_params.copy(),
+            public=self._metadata_public,
+        )["metadata"]
 
     def gallery(self, username, folder_id, offset=0, extend=True, public=None):
         """Yield all Deviation-objects contained in a gallery folder"""
@@ -1411,6 +1449,8 @@ class DeviantartOAuthAPI():
         warn = True
         if public is None:
             public = self.public
+        if self.limit and params["limit"] > self.limit:
+            params["limit"] = self.limit
 
         while True:
             data = self._call(endpoint, params=params, public=public)

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.26.8"
+__version__ = "1.26.9-dev"

test/results/deviantart.py

@@ -642,6 +642,37 @@ __tests__ = (
     "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
 },
 
+{
+    "#url"     : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852",
+    "#category": ("", "deviantart", "deviation"),
+    "#class"   : deviantart.DeviantartDeviationExtractor,
+    "#options" : {"metadata": "submission,camera,stats"},
+
+    "can_post_comment": False,
+    "description"     : str,
+    "is_watching"     : False,
+    "license"         : "No License",
+    "stats": {
+        "comments"       : int,
+        "downloads"      : int,
+        "downloads_today": int,
+        "favourites"     : int,
+        "views"          : int,
+        "views_today"    : int,
+    },
+    "submission": {
+        "category"      : "traditional/drawings/other",
+        "creation_time" : "2004-08-25T02:44:08-0700",
+        "file_size"     : "133 KB",
+        "resolution"    : "710x510",
+        "submitted_with": {
+            "app": "Unknown App",
+            "url": ""
+        },
+    },
+    "tags": [],
+},
+
 {
     "#url"     : "https://www.deviantart.com/zzz/art/zzz-1234567890",
     "#category": ("", "deviantart", "deviation"),