|
|
@ -1,6 +1,6 @@
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
|
|
|
|
# Copyright 2015-2020 Mike Fährmann
|
|
|
|
# Copyright 2015-2021 Mike Fährmann
|
|
|
|
#
|
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
@ -111,10 +111,6 @@ class Job():
|
|
|
|
if self.pred_queue(url, kwds):
|
|
|
|
if self.pred_queue(url, kwds):
|
|
|
|
self.handle_queue(url, kwds)
|
|
|
|
self.handle_queue(url, kwds)
|
|
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Metadata:
|
|
|
|
|
|
|
|
self.update_kwdict(msg[1])
|
|
|
|
|
|
|
|
self.handle_metadata(msg[1])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elif msg[0] == Message.Version:
|
|
|
|
elif msg[0] == Message.Version:
|
|
|
|
if msg[1] != 1:
|
|
|
|
if msg[1] != 1:
|
|
|
|
raise "unsupported message-version ({}, {})".format(
|
|
|
|
raise "unsupported message-version ({}, {})".format(
|
|
|
@ -128,9 +124,6 @@ class Job():
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
"""Handle Message.Directory"""
|
|
|
|
"""Handle Message.Directory"""
|
|
|
|
|
|
|
|
|
|
|
|
def handle_metadata(self, kwdict):
|
|
|
|
|
|
|
|
"""Handle Message.Metadata"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
"""Handle Message.Queue"""
|
|
|
|
"""Handle Message.Queue"""
|
|
|
|
|
|
|
|
|
|
|
@ -280,15 +273,6 @@ class DownloadJob(Job):
|
|
|
|
for callback in self.hooks["post"]:
|
|
|
|
for callback in self.hooks["post"]:
|
|
|
|
callback(self.pathfmt)
|
|
|
|
callback(self.pathfmt)
|
|
|
|
|
|
|
|
|
|
|
|
def handle_metadata(self, kwdict):
|
|
|
|
|
|
|
|
"""Run postprocessors with metadata from 'kwdict'"""
|
|
|
|
|
|
|
|
if "metadata" in self.hooks:
|
|
|
|
|
|
|
|
kwdict["extension"] = "metadata"
|
|
|
|
|
|
|
|
pathfmt = self.pathfmt
|
|
|
|
|
|
|
|
pathfmt.set_filename(kwdict)
|
|
|
|
|
|
|
|
for callback in self.hooks["metadata"]:
|
|
|
|
|
|
|
|
callback(pathfmt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
if url in self.visited:
|
|
|
|
if url in self.visited:
|
|
|
|
return
|
|
|
|
return
|
|
|
@ -624,8 +608,5 @@ class DataJob(Job):
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
def handle_directory(self, kwdict):
|
|
|
|
self.data.append((Message.Directory, self.filter(kwdict)))
|
|
|
|
self.data.append((Message.Directory, self.filter(kwdict)))
|
|
|
|
|
|
|
|
|
|
|
|
def handle_metadata(self, kwdict):
|
|
|
|
|
|
|
|
self.data.append((Message.Metadata, self.filter(kwdict)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
|
|
self.data.append((Message.Queue, url, self.filter(kwdict)))
|
|
|
|
self.data.append((Message.Queue, url, self.filter(kwdict)))
|
|
|
|