diff --git a/gallery_dl/cloudflare.py b/gallery_dl/cloudflare.py index 04c96a83..e464bc4d 100644 --- a/gallery_dl/cloudflare.py +++ b/gallery_dl/cloudflare.py @@ -13,6 +13,7 @@ import time import operator import urllib.parse from . import text +from .cache import memcache def is_challenge(response): @@ -47,7 +48,14 @@ def solve_challenge(session, response, kwargs): location = cf_response.headers["Location"] if location[0] == "/": location = root + location - return location + + for cookie in cf_response.cookies: + if cookie.name == "cf_clearance": + return location, cookie.domain, { + cookie.name: cookie.value, + "__cfduid" : response.cookies.get("__cfduid", ""), + } + return location, "", {} def solve_js_challenge(page, netloc): @@ -126,3 +134,8 @@ VALUES = { "!+": 1, "+!!": 1, } + + +@memcache(keyarg=0) +def cookies(category): + return None diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 3e7e9c47..066bc7fe 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -90,7 +90,9 @@ class Extractor(): return response if cloudflare.is_challenge(response): self.log.info("Solving Cloudflare challenge") - url = cloudflare.solve_challenge(session, response, kwargs) + url, domain, cookies = cloudflare.solve_challenge( + session, response, kwargs) + cloudflare.cookies.update(self.category, (domain, cookies)) continue msg = "{}: {} for url: {}".format(code, response.reason, url) @@ -159,6 +161,11 @@ class Extractor(): else: self.session.cookies.update(cookiejar) + cookies = cloudflare.cookies(self.category) + if cookies: + domain, cookies = cookies + self._update_cookies_dict(cookies, domain) + def _update_cookies(self, cookies, *, domain=""): """Update the session's cookiejar with 'cookies'""" if isinstance(cookies, dict):