From f622411be5e080060c450c8d7129cbb9f1f66527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 13 Mar 2017 21:42:16 +0100 Subject: [PATCH] [deviantart] implement 'skip' method --- gallery_dl/extractor/deviantart.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index b2c03084..a4debdf3 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -30,11 +30,16 @@ class DeviantartUserExtractor(Extractor): Extractor.__init__(self) self.api = DeviantartAPI(self) self.user = match.group(1) + self.offset = 0 + + def skip(self, num): + self.offset += num + return num def items(self): first = True yield Message.Version, 1 - for deviation in self.api.gallery_all(self.user): + for deviation in self.api.gallery_all(self.user, self.offset): if "content" not in deviation: continue if first: @@ -151,6 +156,7 @@ class DeviantartAPI(): params["offset"] = data["next_offset"] else: self.log.error("Unexpected API response: %s", data) + return def authenticate(self): """Authenticate the application by requesting a bearer token""" @@ -175,12 +181,12 @@ class DeviantartAPI(): def _call(self, url, params={}): """Call an API endpoint""" - self.authenticate() - tries = 0 + tries = 1 while True: if self.delay: time.sleep(self.delay) + self.authenticate() response = self.session.get(url, params=params) if response.status_code == 200: @@ -190,8 +196,10 @@ class DeviantartAPI(): self.log.debug("rate limit (delay: %d)", self.delay) else: self.delay = 1 + self.log.debug("http status code %d (%d/3)", + response.status_code, tries) tries += 1 - if tries >= 3: + if tries > 3: raise Exception(response.text) try: return response.json()