[deviantart] implement 'skip' method

pull/13/head
Mike Fährmann 8 years ago
parent 27ae152f57
commit f622411be5
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -30,11 +30,16 @@ class DeviantartUserExtractor(Extractor):
Extractor.__init__(self) Extractor.__init__(self)
self.api = DeviantartAPI(self) self.api = DeviantartAPI(self)
self.user = match.group(1) self.user = match.group(1)
self.offset = 0
def skip(self, num):
self.offset += num
return num
def items(self): def items(self):
first = True first = True
yield Message.Version, 1 yield Message.Version, 1
for deviation in self.api.gallery_all(self.user): for deviation in self.api.gallery_all(self.user, self.offset):
if "content" not in deviation: if "content" not in deviation:
continue continue
if first: if first:
@ -151,6 +156,7 @@ class DeviantartAPI():
params["offset"] = data["next_offset"] params["offset"] = data["next_offset"]
else: else:
self.log.error("Unexpected API response: %s", data) self.log.error("Unexpected API response: %s", data)
return
def authenticate(self): def authenticate(self):
"""Authenticate the application by requesting a bearer token""" """Authenticate the application by requesting a bearer token"""
@ -175,12 +181,12 @@ class DeviantartAPI():
def _call(self, url, params={}): def _call(self, url, params={}):
"""Call an API endpoint""" """Call an API endpoint"""
self.authenticate() tries = 1
tries = 0
while True: while True:
if self.delay: if self.delay:
time.sleep(self.delay) time.sleep(self.delay)
self.authenticate()
response = self.session.get(url, params=params) response = self.session.get(url, params=params)
if response.status_code == 200: if response.status_code == 200:
@ -190,8 +196,10 @@ class DeviantartAPI():
self.log.debug("rate limit (delay: %d)", self.delay) self.log.debug("rate limit (delay: %d)", self.delay)
else: else:
self.delay = 1 self.delay = 1
self.log.debug("http status code %d (%d/3)",
response.status_code, tries)
tries += 1 tries += 1
if tries >= 3: if tries > 3:
raise Exception(response.text) raise Exception(response.text)
try: try:
return response.json() return response.json()

Loading…
Cancel
Save