|
|
|
@ -110,12 +110,6 @@ class Job():
|
|
|
|
|
if self.pred_queue(url, kwds):
|
|
|
|
|
self.handle_queue(url, kwds)
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Urllist:
|
|
|
|
|
_, urls, kwds = msg
|
|
|
|
|
if self.pred_url(urls[0], kwds):
|
|
|
|
|
self.update_kwdict(kwds)
|
|
|
|
|
self.handle_urllist(urls, kwds)
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Metadata:
|
|
|
|
|
self.update_kwdict(msg[1])
|
|
|
|
|
self.handle_metadata(msg[1])
|
|
|
|
@ -130,10 +124,6 @@ class Job():
|
|
|
|
|
def handle_url(self, url, kwdict):
|
|
|
|
|
"""Handle Message.Url"""
|
|
|
|
|
|
|
|
|
|
def handle_urllist(self, urls, kwdict):
|
|
|
|
|
"""Handle Message.Urllist"""
|
|
|
|
|
self.handle_url(urls[0], kwdict)
|
|
|
|
|
|
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
|
"""Handle Message.Directory"""
|
|
|
|
|
|
|
|
|
@ -215,7 +205,7 @@ class DownloadJob(Job):
|
|
|
|
|
else:
|
|
|
|
|
self.visited = set()
|
|
|
|
|
|
|
|
|
|
def handle_url(self, url, kwdict, fallback=None):
|
|
|
|
|
def handle_url(self, url, kwdict):
|
|
|
|
|
"""Download the resource specified in 'url'"""
|
|
|
|
|
postprocessors = self.postprocessors
|
|
|
|
|
pathfmt = self.pathfmt
|
|
|
|
@ -246,7 +236,7 @@ class DownloadJob(Job):
|
|
|
|
|
if not self.download(url):
|
|
|
|
|
|
|
|
|
|
# use fallback URLs if available
|
|
|
|
|
for num, url in enumerate(fallback or (), 1):
|
|
|
|
|
for num, url in enumerate(kwdict.get("_fallback", ()), 1):
|
|
|
|
|
util.remove_file(pathfmt.temppath)
|
|
|
|
|
self.log.info("Trying fallback URL #%d", num)
|
|
|
|
|
if self.download(url):
|
|
|
|
@ -279,12 +269,6 @@ class DownloadJob(Job):
|
|
|
|
|
pp.run_after(pathfmt)
|
|
|
|
|
self._skipcnt = 0
|
|
|
|
|
|
|
|
|
|
def handle_urllist(self, urls, kwdict):
|
|
|
|
|
"""Download the resource specified in 'url'"""
|
|
|
|
|
fallback = iter(urls)
|
|
|
|
|
url = next(fallback)
|
|
|
|
|
self.handle_url(url, kwdict, fallback)
|
|
|
|
|
|
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
|
"""Set and create the target directory for downloads"""
|
|
|
|
|
if not self.pathfmt:
|
|
|
|
@ -563,15 +547,11 @@ class UrlJob(Job):
|
|
|
|
|
self.handle_queue = self.handle_url
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def handle_url(url, _):
|
|
|
|
|
def handle_url(url, kwdict):
|
|
|
|
|
print(url)
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def handle_urllist(urls, _):
|
|
|
|
|
prefix = ""
|
|
|
|
|
for url in urls:
|
|
|
|
|
print(prefix, url, sep="")
|
|
|
|
|
prefix = "| "
|
|
|
|
|
if "_fallback" in kwdict:
|
|
|
|
|
for url in kwdict["_fallback"]:
|
|
|
|
|
print("|", url)
|
|
|
|
|
|
|
|
|
|
def handle_queue(self, url, _):
|
|
|
|
|
try:
|
|
|
|
@ -625,9 +605,6 @@ class DataJob(Job):
|
|
|
|
|
def handle_url(self, url, kwdict):
|
|
|
|
|
self.data.append((Message.Url, url, self.filter(kwdict)))
|
|
|
|
|
|
|
|
|
|
def handle_urllist(self, urls, kwdict):
|
|
|
|
|
self.data.append((Message.Urllist, list(urls), self.filter(kwdict)))
|
|
|
|
|
|
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
|
self.data.append((Message.Directory, self.filter(kwdict)))
|
|
|
|
|
|
|
|
|
|