add 'sleep-extractor' option (closes #964)

(would have been nice if this were possible without code duplication)
pull/997/head
Mike Fährmann 4 years ago
parent 3108e85b89
commit 3afd362e2e
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -208,6 +208,16 @@ Description Number of seconds to sleep before each download.
=========== =====
extractor.*.sleep-extractor
---------------------------
=========== =====
Type ``float``
Default ``0``
Description Number of seconds to sleep before handling an input URL,
i.e. before starting a new extractor.
=========== =====
extractor.*.username & .password
--------------------------------
=========== =====

@ -59,6 +59,9 @@ class Job():
def run(self):
"""Execute or run the job"""
sleep = self.extractor.config("sleep-extractor")
if sleep:
time.sleep(sleep)
try:
log = self.extractor.log
for msg in self.extractor:
@ -586,6 +589,10 @@ class DataJob(Job):
self.filter = (lambda x: x) if private else util.filter_dict
def run(self):
sleep = self.extractor.config("sleep-extractor")
if sleep:
time.sleep(sleep)
# collect data
try:
for msg in self.extractor:

Loading…
Cancel
Save