diff --git a/docs/configuration.rst b/docs/configuration.rst index 2c1b3aea..a6e1e9f0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -218,6 +218,16 @@ Description Number of seconds to sleep before handling an input URL, =========== ===== +extractor.*.sleep-request +------------------------- +=========== ===== +Type ``float`` +Default ``0`` +Description Minimal time interval in seconds between each HTTP request + during data extraction. +=========== ===== + + extractor.*.username & .password -------------------------------- =========== ===== diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 7f787471..357deac8 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -31,6 +31,8 @@ class Extractor(): cookiedomain = "" root = "" test = None + _request_last = 0 + _request_interval = 0 def __init__(self, match): self.session = requests.Session() @@ -46,6 +48,8 @@ class Extractor(): self._retries = self.config("retries", 4) self._timeout = self.config("timeout", 30) self._verify = self.config("verify", True) + self._request_interval = self.config( + "sleep-request", self._request_interval) if self._retries < 0: self._retries = float("inf") @@ -85,6 +89,13 @@ class Extractor(): kwargs.setdefault("verify", self._verify) response = None + if self._request_interval: + seconds = (self._request_interval - + (time.time() - Extractor._request_last)) + if seconds > 0: + self.log.debug("Sleeping for %.5s seconds", seconds) + time.sleep(seconds) + while True: try: response = session.request(method, url, **kwargs) @@ -123,6 +134,8 @@ class Extractor(): msg = "'{} {}' for '{}'".format(code, reason, url) if code < 500 and code != 429 and code != 430: break + finally: + Extractor._request_last = time.time() self.log.debug("%s (%s/%s)", msg, tries, retries+1) if tries > retries: