[deviantart] try to work around journal/status API changes (#5916)

The new /user/profile/posts endpoint is documented to "return all journals & status updates for a given user in a single feed" but it does not do this. There are usually several or even all status updates missing.
2 months ago · bb07313227
parent f87783ff26
commit bb07313227
3 changed files with 43 additions and 128 deletions
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@ -178,7 +178,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
    <td>DeviantArt</td>
    <td>https://www.deviantart.com/</td>
-    <td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
+    <td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
    <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
 </tr>
 <tr>
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@ -846,55 +846,6 @@ class DeviantartStatusExtractor(DeviantartExtractor):
            )


-class DeviantartPopularExtractor(DeviantartExtractor):
-    """Extractor for popular deviations"""
-    subcategory = "popular"
-    directory_fmt = ("{category}", "Popular",
-                     "{popular[range]}", "{popular[search]}")
-    archive_fmt = "P_{popular[range]}_{popular[search]}_{index}.{extension}"
-    pattern = (r"(?:https?://)?www\.deviantart\.com/(?:"
-               r"(?:deviations/?)?\?order=(popular-[^/?#]+)"
-               r"|((?:[\w-]+/)*)(popular-[^/?#]+)"
-               r")/?(?:\?([^#]*))?")
-    example = "https://www.deviantart.com/popular-24-hours/"
-
-    def __init__(self, match):
-        DeviantartExtractor.__init__(self, match)
-        self.user = ""
-
-        trange1, path, trange2, query = match.groups()
-        query = text.parse_query(query)
-        self.search_term = query.get("q")
-
-        trange = trange1 or trange2 or query.get("order", "")
-        if trange.startswith("popular-"):
-            trange = trange[8:]
-        self.time_range = {
-            "newest"      : "now",
-            "most-recent" : "now",
-            "this-week"   : "1week",
-            "this-month"  : "1month",
-            "this-century": "alltime",
-            "all-time"    : "alltime",
-        }.get(trange, "alltime")
-
-        self.popular = {
-            "search": self.search_term or "",
-            "range" : trange or "all-time",
-            "path"  : path.strip("/") if path else "",
-        }
-
-    def deviations(self):
-        if self.time_range == "now":
-            return self.api.browse_newest(self.search_term, self.offset)
-        return self.api.browse_popular(
-            self.search_term, self.time_range, self.offset)
-
-    def prepare(self, deviation):
-        DeviantartExtractor.prepare(self, deviation)
-        deviation["popular"] = self.popular
-
-
 class DeviantartTagExtractor(DeviantartExtractor):
    """Extractor for deviations from tag searches"""
    subcategory = "tag"
@ -1095,7 +1046,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor):
 class DeviantartOAuthAPI():
    """Interface for the DeviantArt OAuth API

-    Ref: https://www.deviantart.com/developers/http/v1/20160316
+    https://www.deviantart.com/developers/http/v1/20160316
    """
    CLIENT_ID = "5388"
    CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
@ -1188,29 +1139,6 @@ class DeviantartOAuthAPI():
                  "mature_content": self.mature}
        return self._pagination(endpoint, params, public=False, unpack=True)

-    def browse_newest(self, query=None, offset=0):
-        """Browse newest deviations"""
-        endpoint = "/browse/newest"
-        params = {
-            "q"             : query,
-            "limit"         : 120,
-            "offset"        : offset,
-            "mature_content": self.mature,
-        }
-        return self._pagination(endpoint, params)
-
-    def browse_popular(self, query=None, timerange=None, offset=0):
-        """Yield popular deviations"""
-        endpoint = "/browse/popular"
-        params = {
-            "q"             : query,
-            "limit"         : 120,
-            "timerange"     : timerange,
-            "offset"        : offset,
-            "mature_content": self.mature,
-        }
-        return self._pagination(endpoint, params)
-
    def browse_tags(self, tag, offset=0):
        """ Browse a tag """
        endpoint = "/browse/tags"
@ -1223,11 +1151,12 @@ class DeviantartOAuthAPI():
        return self._pagination(endpoint, params)

    def browse_user_journals(self, username, offset=0):
-        """Yield all journal entries of a specific user"""
-        endpoint = "/browse/user/journals"
-        params = {"username": username, "offset": offset, "limit": 50,
-                  "mature_content": self.mature, "featured": "false"}
-        return self._pagination(endpoint, params)
+        journals = filter(
+            lambda post: "/journal/" in post["url"],
+            self.user_profile_posts(username))
+        if offset:
+            journals = util.advance(journals, offset)
+        return journals

    def collections(self, username, folder_id, offset=0):
        """Yield all Deviation-objects contained in a collection folder"""
@ -1339,18 +1268,6 @@ class DeviantartOAuthAPI():
                  "mature_content": self.mature}
        return self._pagination_list(endpoint, params)

-    @memcache(keyarg=1)
-    def user_profile(self, username):
-        """Get user profile information"""
-        endpoint = "/user/profile/" + username
-        return self._call(endpoint, fatal=False)
-
-    def user_statuses(self, username, offset=0):
-        """Yield status updates of a specific user"""
-        endpoint = "/user/statuses/"
-        params = {"username": username, "offset": offset, "limit": 50}
-        return self._pagination(endpoint, params)
-
    def user_friends(self, username, offset=0):
        """Get the users list of friends"""
        endpoint = "/user/friends/" + username
@ -1382,6 +1299,27 @@ class DeviantartOAuthAPI():
            endpoint, method="POST", public=False, fatal=False,
        ).get("success")

+    @memcache(keyarg=1)
+    def user_profile(self, username):
+        """Get user profile information"""
+        endpoint = "/user/profile/" + username
+        return self._call(endpoint, fatal=False)
+
+    def user_profile_posts(self, username):
+        endpoint = "/user/profile/posts"
+        params = {"username": username, "limit": 50,
+                  "mature_content": self.mature}
+        return self._pagination(endpoint, params)
+
+    def user_statuses(self, username, offset=0):
+        """Yield status updates of a specific user"""
+        statuses = filter(
+            lambda post: "/status-update/" in post["url"],
+            self.user_profile_posts(username))
+        if offset:
+            statuses = util.advance(statuses, offset)
+        return statuses
+
    def authenticate(self, refresh_token_key):
        """Authenticate the application by requesting an access token"""
        self.headers["Authorization"] = \
@ -1470,7 +1408,7 @@ class DeviantartOAuthAPI():
                    self.log.error(msg)
                return data

-    def _switch_tokens(self, results, params):
+    def _should_switch_tokens(self, results, params):
        if len(results) < params["limit"]:
            return True

@ -1502,7 +1440,7 @@ class DeviantartOAuthAPI():
                results = [item["journal"] for item in results
                           if "journal" in item]
            if extend:
-                if public and self._switch_tokens(results, params):
+                if public and self._should_switch_tokens(results, params):
                    if self.refresh_token_key:
                        self.log.debug("Switching to private access token")
                        public = False
@ -1546,6 +1484,11 @@ class DeviantartOAuthAPI():
                    return
                params["offset"] = int(params["offset"]) + len(results)

+    def _pagination_list(self, endpoint, params, key="results"):
+        result = []
+        result.extend(self._pagination(endpoint, params, False, key=key))
+        return result
+
    @staticmethod
    def _shared_content(results):
        """Return an iterable of shared deviations in 'results'"""
@ -1554,11 +1497,6 @@ class DeviantartOAuthAPI():
                if "deviation" in item:
                    yield item["deviation"]

-    def _pagination_list(self, endpoint, params, key="results"):
-        result = []
-        result.extend(self._pagination(endpoint, params, False, key=key))
-        return result
-
    def _metadata(self, deviations):
        """Add extended metadata to each deviation object"""
        if len(deviations) <= self.limit:
--- a/test/results/deviantart.py
+++ b/test/results/deviantart.py
@ -48,7 +48,7 @@ __tests__ = (

    "allows_comments" : bool,
    "author"          : {
-        "type"    : "regular",
+        "type"    : "premium",
        "usericon": str,
        "userid"  : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
        "username": "shimoda7",
@ -210,7 +210,7 @@ __tests__ = (
    "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f",

    "author"         : {
-        "type"    : "regular",
+        "type"    : "premium",
        "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4",
        "userid"  : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
        "username": "shimoda7",
@ -502,7 +502,7 @@ __tests__ = (
    "#url"     : "https://www.deviantart.com/angrywhitewanker/posts/journals/",
    "#category": ("", "deviantart", "journal"),
    "#class"   : deviantart.DeviantartJournalExtractor,
-    "#sha1_url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44",
+    "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5",
 },

 {
@ -616,30 +616,6 @@ __tests__ = (
    "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c",
 },

-{
-    "#url"     : "https://www.deviantart.com/?order=popular-all-time",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-    "#options" : {"original": False},
-    "#range"   : "1-30",
-    "#count"   : 30,
-},
-
-{
-    "#url"     : "https://www.deviantart.com/popular-24-hours/?q=tree+house",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-    "#options" : {"original": False},
-    "#range"   : "1-30",
-    "#count"   : 30,
-},
-
-{
-    "#url"     : "https://www.deviantart.com/artisan/popular-all-time/?q=tree",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-},
-
 {
    "#url"     : "https://www.deviantart.com/tag/nature",
    "#category": ("", "deviantart", "tag"),
@ -811,7 +787,7 @@ __tests__ = (
    "#category": ("", "deviantart", "deviation"),
    "#class"   : deviantart.DeviantartDeviationExtractor,
    "#pattern" : """text:<!DOCTYPE html>\n""",
-    "#sha1_url": "d34b2c9f873423e665a1b8ced20fcb75951694a3",
+    "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c",
 },

 {
@ -820,7 +796,7 @@ __tests__ = (
    "#category": ("", "deviantart", "deviation"),
    "#class"   : deviantart.DeviantartDeviationExtractor,
    "#pattern" : """text:<!DOCTYPE html>\n""",
-    "#sha1_url": "e2e0044bd255304412179b6118536dbd9bb3bb0e",
+    "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469",
 },

 {
@ -829,6 +805,7 @@ __tests__ = (
    "#category": ("", "deviantart", "deviation"),
    "#class"   : deviantart.DeviantartDeviationExtractor,
    "#count"   : 0,
+    "#exception": exception.NotFoundError,
 },

 {
@ -866,7 +843,7 @@ __tests__ = (
    "#url"     : "https://www.deviantart.com/view/706871727",
    "#category": ("", "deviantart", "deviation"),
    "#class"   : deviantart.DeviantartDeviationExtractor,
-    "#sha1_content": "87dff6056fc9a2bf77f75317a1e00e18451b3c80",
+    "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856",
 },

 {