|
|
@ -19,7 +19,7 @@ class DownloadManager():
|
|
|
|
def __init__(self, opts, conf):
|
|
|
|
def __init__(self, opts, conf):
|
|
|
|
self.opts = opts
|
|
|
|
self.opts = opts
|
|
|
|
self.conf = conf
|
|
|
|
self.conf = conf
|
|
|
|
self.downloaders = {}
|
|
|
|
self.modules = {}
|
|
|
|
self.extractors = ExtractorFinder(conf)
|
|
|
|
self.extractors = ExtractorFinder(conf)
|
|
|
|
|
|
|
|
|
|
|
|
def add(self, url):
|
|
|
|
def add(self, url):
|
|
|
@ -28,10 +28,10 @@ class DownloadManager():
|
|
|
|
|
|
|
|
|
|
|
|
def get_downloader_module(self, scheme):
|
|
|
|
def get_downloader_module(self, scheme):
|
|
|
|
"""Return a downloader module suitable for 'scheme'"""
|
|
|
|
"""Return a downloader module suitable for 'scheme'"""
|
|
|
|
module = self.downloaders.get(scheme)
|
|
|
|
module = self.modules.get(scheme)
|
|
|
|
if module is None:
|
|
|
|
if module is None:
|
|
|
|
module = importlib.import_module(".downloader."+scheme, __package__)
|
|
|
|
module = importlib.import_module(".downloader."+scheme, __package__)
|
|
|
|
self.downloaders[scheme] = module
|
|
|
|
self.modules[scheme] = module
|
|
|
|
return module
|
|
|
|
return module
|
|
|
|
|
|
|
|
|
|
|
|
def get_base_directory(self):
|
|
|
|
def get_base_directory(self):
|
|
|
@ -58,6 +58,12 @@ class DownloadJob():
|
|
|
|
if msg[0] == Message.Url:
|
|
|
|
if msg[0] == Message.Url:
|
|
|
|
self.download(msg)
|
|
|
|
self.download(msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Headers:
|
|
|
|
|
|
|
|
self.get_downloader("http:").set_headers(msg[1])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Cookies:
|
|
|
|
|
|
|
|
self.get_downloader("http:").set_cookies(msg[1])
|
|
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Directory:
|
|
|
|
elif msg[0] == Message.Directory:
|
|
|
|
self.set_directory(msg)
|
|
|
|
self.set_directory(msg)
|
|
|
|
|
|
|
|
|
|
|
|