initial support for child extractor options

Using "parent-category>child-category" as extractor category in a config
file allows to set options for a child extractor when it was spawned by
that parent.

For example "reddit>gfycat" to set gfycat options for when it was found
in a reddit post.

{
    "extractor": {
        "gfycat": {
            "filename": "regular filename"
        },
        "reddit>gfycat": {
            "filename": "reddit-specific filename"
        }
    }
}

Note: This does currently not work for most imgur links due to how its
extractor hierarchy is structured.
pull/4403/head
Mike Fährmann 1 year ago
parent 255d08b79e
commit ed21908fda
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -45,10 +45,6 @@ class Extractor():
def __init__(self, match):
self.log = logging.getLogger(self.category)
self.url = match.string
if self.basecategory:
self.config = self._config_shared
self.config_accumulate = self._config_shared_accumulate
self._cfgpath = ("extractor", self.category, self.subcategory)
self._parentdir = ""
@ -98,16 +94,22 @@ class Extractor():
return config.accumulate(self._cfgpath, key)
def _config_shared(self, key, default=None):
return config.interpolate_common(("extractor",), (
(self.category, self.subcategory),
(self.basecategory, self.subcategory),
), key, default)
return config.interpolate_common(
("extractor",), self._cfgpath, key, default)
def _config_shared_accumulate(self, key):
values = config.accumulate(self._cfgpath, key)
conf = config.get(("extractor",), self.basecategory)
if conf:
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
first = True
extr = ("extractor",)
for path in self._cfgpath:
if first:
first = False
values = config.accumulate(extr + path, key)
else:
conf = config.get(extr, path[0])
if conf:
values[:0] = config.accumulate(
(self.subcategory,), key, conf=conf)
return values
def request(self, url, method="GET", session=None,

@ -32,6 +32,21 @@ class Job():
self.kwdict = {}
self.status = 0
cfgpath = []
if parent and parent.extractor.category != extr.category:
cat = "{}>{}".format(
parent.extractor.category, extr.category)
cfgpath.append((cat, extr.subcategory))
cfgpath.append((extr.category, extr.subcategory))
if extr.basecategory:
if not cfgpath:
cfgpath.append((extr.category, extr.subcategory))
cfgpath.append((extr.basecategory, extr.subcategory))
if cfgpath:
extr._cfgpath = cfgpath
extr.config = extr._config_shared
extr.config_accumulate = extr._config_shared_accumulate
actions = extr.config("actions")
if actions:
from .actions import parse

Loading…
Cancel
Save