[common] add 'sleep-429' option (#5160)

docs/configuration.rst

@@ -358,13 +358,24 @@ Description
     i.e. before starting a new extractor.
 
 
+extractor.*.sleep-429
+---------------------
+Type
+    |Duration|_
+Default
+    ``60``
+Description
+    Number of seconds to sleep when receiving a `429 Too Many Requests`
+    response before `retrying <extractor.*.retries_>`__ the request.
+
+
 extractor.*.sleep-request
 -------------------------
 Type
     |Duration|_
 Default
     * ``"0.5-1.5"``
-        ``[Danbooru]``, ``[E621]``, ``[foolfuuka]``, ``itaku``,
+        ``[Danbooru]``, ``[E621]``, ``[foolfuuka]:search``, ``itaku``,
         ``newgrounds``, ``[philomena]``, ``pixiv:novel``, ``plurk``,
         ``poipiku`` , ``pornpics``, ``soundgasm``, ``urlgalleries``,
         ``vk``, ``zerochan``

gallery_dl/extractor/common.py

@@ -198,7 +198,10 @@ class Extractor():
                     if b'name="captcha-bypass"' in content:
                         self.log.warning("Cloudflare CAPTCHA")
                         break
-                if code not in retry_codes and code < 500:
+
+                if code == 429 and self._interval_429:
+                    pass
+                elif code not in retry_codes and code < 500:
                     break
 
             finally:
@@ -208,13 +211,17 @@ class Extractor():
             if tries > retries:
                 break
 
-            if self._interval:
-                seconds = self._interval()
-                if seconds < tries:
             seconds = tries
+            if self._interval:
+                s = self._interval()
+                if seconds < s:
+                    seconds = s
+            if code == 429 and self._interval_429:
+                s = self._interval_429()
+                if seconds < s:
+                    seconds = s
+                self.wait(seconds=seconds, reason="429 Too Many Requests")
             else:
-                seconds = tries
-
                 self.sleep(seconds, "retry")
             tries += 1
 
@@ -293,6 +300,9 @@ class Extractor():
             self.config("sleep-request", self.request_interval),
             self.request_interval_min,
         )
+        self._interval_429 = util.build_duration_func(
+            self.config("sleep-429", 60),
+        )
 
         if self._retries < 0:
             self._retries = float("inf")