mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge SharedConfigMixin functionality into Extractor

Browse Source
pull/1195/head
Mike Fährmann 4 years ago
parent ddfb4fd07a
commit 1e3dd7330e
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
11 changed files with 40 additions and 45 deletions
Show Stats Download Patch File Download Diff File

4
gallery_dl/extractor/booru.py
Unescape View File

@ -8,7 +8,7 @@
"""Base classes for extractors for danbooru and co"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text, exception
from xml.etree import ElementTree
import collections
@ -17,7 +17,7 @@ import operator
import re
class BooruExtractor(SharedConfigMixin, Extractor):
class BooruExtractor(Extractor):
"""Base class for all booru extractors"""
basecategory = "booru"
filename_fmt = "{category}_{id}_{md5}.{extension}"

40
gallery_dl/extractor/common.py
Unescape View File

@ -24,6 +24,7 @@ class Extractor():
category = ""
subcategory = ""
basecategory = ""
categorytransfer = False
directory_fmt = ("{category}",)
filename_fmt = "{filename}.{extension}"
@ -54,6 +55,10 @@ class Extractor():
if self._retries < 0:
self._retries = float("inf")
if self.basecategory:
self.config = self._config_shared
self.config_accumulate = self._config_shared_accumulate
self._init_headers()
self._init_cookies()
self._init_proxies()
@ -80,6 +85,19 @@ class Extractor():
def config_accumulate(self, key):
return config.accumulate(self._cfgpath, key)
def _config_shared(self, key, default=None):
return config.interpolate_common(("extractor",), (
(self.category, self.subcategory),
(self.basecategory, self.subcategory),
), key, default)
def _config_shared_accumulate(self, key):
values = config.accumulate(self._cfgpath, key)
conf = config.get(("extractor",), self.basecategory)
if conf:
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
return values
def request(self, url, *, method="GET", session=None, retries=None,
encoding=None, fatal=True, notfound=None, **kwargs):
tries = 1
@ -506,28 +524,6 @@ class AsynchronousMixin():
messages.put(None)
class SharedConfigMixin():
"""Enable sharing of config settings based on 'basecategory'"""
basecategory = ""
def config(self, key, default=None):
return config.interpolate_common(
("extractor",), (
(self.category, self.subcategory),
(self.basecategory, self.subcategory),
), key, default,
)
def config_accumulate(self, key):
values = config.accumulate(self._cfgpath, key)
conf = config.get(("extractor",), self.basecategory)
if conf:
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
return values
def generate_extractors(extractor_data, symtable, classes):
"""Dynamically generate Extractor classes"""
extractors = config.get(("extractor",), classes[0].basecategory)

4
gallery_dl/extractor/danbooru.py
Unescape View File

@ -8,7 +8,7 @@
"""Extractors for https://danbooru.donmai.us/"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text
import datetime
@ -20,7 +20,7 @@ BASE_PATTERN = (
)
class DanbooruExtractor(SharedConfigMixin, Extractor):
class DanbooruExtractor(Extractor):
"""Base class for danbooru extractors"""
basecategory = "booru"
category = "danbooru"

4
gallery_dl/extractor/foolfuuka.py
Unescape View File

@ -8,13 +8,13 @@
"""Extractors for 4chan archives based on FoolFuuka"""
from .common import Extractor, Message, SharedConfigMixin, generate_extractors
from .common import Extractor, Message, generate_extractors
from .. import text
import itertools
import operator
class FoolfuukaThreadExtractor(SharedConfigMixin, Extractor):
class FoolfuukaThreadExtractor(Extractor):
"""Base extractor for FoolFuuka based boards/archives"""
basecategory = "foolfuuka"
subcategory = "thread"

3
gallery_dl/extractor/foolslide.py
Unescape View File

@ -12,7 +12,6 @@ from .common import (
Extractor,
ChapterExtractor,
MangaExtractor,
SharedConfigMixin,
Message,
generate_extractors,
)
@ -20,7 +19,7 @@ from .. import text, util
import json
class FoolslideBase(SharedConfigMixin):
class FoolslideBase():
"""Base class for FoOlSlide extractors"""
basecategory = "foolslide"

6
gallery_dl/extractor/imagehosts.py
Unescape View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016-2019 Mike Fährmann
# Copyright 2016-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@ -8,13 +8,13 @@
"""Collection of extractors for various imagehosts"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text, exception
from ..cache import memcache
from os.path import splitext
class ImagehostImageExtractor(SharedConfigMixin, Extractor):
class ImagehostImageExtractor(Extractor):
"""Base class for single-image extractors for various imagehosts"""
basecategory = "imagehost"
subcategory = "image"

6
gallery_dl/extractor/paheal.py
Unescape View File

@ -6,13 +6,13 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from https://rule34.paheal.net/"""
"""Extractors for https://rule34.paheal.net/"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text
class PahealExtractor(SharedConfigMixin, Extractor):
class PahealExtractor(Extractor):
"""Base class for paheal extractors"""
basecategory = "booru"
category = "paheal"

6
gallery_dl/extractor/reactor.py
Unescape View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann
# Copyright 2019-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@ -8,7 +8,7 @@
"""Generic extractors for *reactor sites"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text
import urllib.parse
import random
@ -19,7 +19,7 @@ import json
BASE_PATTERN = r"(?:https?://)?((?:[^/.]+\.)?reactor\.cc)"
class ReactorExtractor(SharedConfigMixin, Extractor):
class ReactorExtractor(Extractor):
"""Base class for *reactor.cc extractors"""
basecategory = "reactor"
filename_fmt = "{post_id}_{num:>02}{title[:100]:?_//}.{extension}"

6
gallery_dl/extractor/sankaku.py
Unescape View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2019 Mike Fährmann
# Copyright 2014-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@ -8,7 +8,7 @@
"""Extractors for https://chan.sankakucomplex.com/"""
from .common import Extractor, Message, SharedConfigMixin
from .common import Extractor, Message
from .. import text, util, exception
from ..cache import cache
import collections
@ -17,7 +17,7 @@ import time
import re
class SankakuExtractor(SharedConfigMixin, Extractor):
class SankakuExtractor(Extractor):
"""Base class for sankaku extractors"""
basecategory = "booru"
category = "sankaku"

4
gallery_dl/extractor/shopify.py
Unescape View File

@ -8,12 +8,12 @@
"""Extractors for Shopify instances"""
from .common import Extractor, Message, SharedConfigMixin, generate_extractors
from .common import Extractor, Message, generate_extractors
from .. import text
import re
class ShopifyExtractor(SharedConfigMixin, Extractor):
class ShopifyExtractor(Extractor):
"""Base class for Shopify extractors"""
basecategory = "shopify"
filename_fmt = "{product[title]}_{num:>02}_{id}.{extension}"

2
test/test_results.py
Unescape View File

@ -368,7 +368,7 @@ def generate_tests():
# filter available extractor classes
extractors = [
extr for extr in extractor.extractors()
if fltr(extr.category, getattr(extr, "basecategory", None))
if fltr(extr.category, extr.basecategory)
]
# add 'test_...' methods

Write Preview
Loading…
Cancel
Save