fix issues with 'Extractor.finalize()'

- prevent crash in InstagramUserExtractor (#4359)
- call it at the end of every DownloadJob
- add it to tests
pull/4403/head
Mike Fährmann 1 year ago
parent ed21908fda
commit 48ef062867
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -36,7 +36,6 @@ class Extractor():
browser = None browser = None
root = "" root = ""
test = None test = None
finalize = None
request_interval = 0.0 request_interval = 0.0
request_interval_min = 0.0 request_interval_min = 0.0
request_timestamp = 0.0 request_timestamp = 0.0
@ -66,6 +65,9 @@ class Extractor():
self._init() self._init()
self.initialize = util.noop self.initialize = util.noop
def finalize(self):
pass
def items(self): def items(self):
yield Message.Version, 1 yield Message.Version, 1

@ -405,6 +405,9 @@ class InstagramUserExtractor(InstagramExtractor):
def initialize(self): def initialize(self):
pass pass
def finalize(self):
pass
def items(self): def items(self):
base = "{}/{}/".format(self.root, self.item) base = "{}/{}/".format(self.root, self.item)
stories = "{}/stories/{}/".format(self.root, self.item) stories = "{}/stories/{}/".format(self.root, self.item)

@ -140,7 +140,6 @@ class Job():
log.info("No results for %s", extractor.url) log.info("No results for %s", extractor.url)
finally: finally:
self.handle_finalize() self.handle_finalize()
if extractor.finalize:
extractor.finalize() extractor.finalize()
return self.status return self.status

@ -138,6 +138,7 @@ class TestExtractorModule(unittest.TestCase):
for test in cls._get_tests(): for test in cls._get_tests():
extr = cls.from_url(test[0]) extr = cls.from_url(test[0])
extr.initialize() extr.initialize()
extr.finalize()
break break
def test_docstrings(self): def test_docstrings(self):

Loading…
Cancel
Save