diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 0dd05ef2..cf0f8c90 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -102,6 +102,9 @@ class Extractor(): def config_accumulate(self, key): return config.accumulate(self._cfgpath, key) + def config_instance(self, key, default=None): + return default + def _config_shared(self, key, default=None): return config.interpolate_common( ("extractor",), self._cfgpath, key, default) @@ -735,9 +738,10 @@ class BaseExtractor(Extractor): for index, group in enumerate(match.groups()): if group is not None: if index: - self.category, self.root = self.instances[index-1] + self.category, self.root, info = self.instances[index-1] if not self.root: self.root = text.root_from_url(match.group(0)) + self.config_instance = info.get else: self.root = group self.category = group.partition("://")[2] @@ -757,7 +761,7 @@ class BaseExtractor(Extractor): root = info["root"] if root: root = root.rstrip("/") - instance_list.append((category, root)) + instance_list.append((category, root, info)) pattern = info.get("pattern") if not pattern: diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index 0c8af3d5..f8ab71c9 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -22,11 +22,7 @@ class GelbooruV02Extractor(booru.BooruExtractor): def _init(self): self.api_key = self.config("api-key") self.user_id = self.config("user-id") - - try: - self.api_root = INSTANCES[self.category]["api_root"] - except KeyError: - self.api_root = self.root + self.api_root = self.config_instance("api_root") or self.root if self.category == "realbooru": self.items = self._items_realbooru @@ -161,7 +157,7 @@ class GelbooruV02Extractor(booru.BooruExtractor): post["tags_" + key] = " ".join(value) -INSTANCES = { +BASE_PATTERN = GelbooruV02Extractor.update({ "realbooru": { "root": "https://realbooru.com", "pattern": r"realbooru\.com", @@ -187,9 +183,7 @@ INSTANCES = { "root": "https://xbooru.com", "pattern": r"xbooru\.com", }, -} - -BASE_PATTERN = GelbooruV02Extractor.update(INSTANCES) +}) class GelbooruV02TagExtractor(GelbooruV02Extractor): diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 0b63d6c1..68b41961 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -75,7 +75,7 @@ class MastodonExtractor(BaseExtractor): account["acct"], account["moved"]["acct"]) -INSTANCES = { +BASE_PATTERN = MastodonExtractor.update({ "mastodon.social": { "root" : "https://mastodon.social", "pattern" : r"mastodon\.social", @@ -100,9 +100,7 @@ INSTANCES = { "client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o", "client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY", } -} - -BASE_PATTERN = MastodonExtractor.update(INSTANCES) + "(?:/web)?" +}) + "(?:/web)?" class MastodonUserExtractor(MastodonExtractor): @@ -174,10 +172,8 @@ class MastodonAPI(): if access_token is None or access_token == "cache": access_token = _access_token_cache(extractor.instance) if not access_token: - try: - access_token = INSTANCES[extractor.category]["access-token"] - except (KeyError, TypeError): - pass + access_token = extractor.config_instance("access-token") + if access_token: self.headers = {"Authorization": "Bearer " + access_token} else: diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index 16901607..8c8a5a99 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -358,8 +358,8 @@ class OAuthMastodon(OAuthBase): yield Message.Version, 1 from . import mastodon - for application in mastodon.INSTANCES.values(): - if self.instance == application["root"].partition("://")[2]: + for _, root, application in mastodon.MastodonExtractor.instances: + if self.instance == root.partition("://")[2]: break else: application = self._register(self.instance) diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index ac6a391e..339646ff 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -32,7 +32,7 @@ class PhilomenaExtractor(BooruExtractor): post["date"] = text.parse_datetime(post["created_at"]) -INSTANCES = { +BASE_PATTERN = PhilomenaExtractor.update({ "derpibooru": { "root": "https://derpibooru.org", "pattern": r"(?:www\.)?derpibooru\.org", @@ -48,9 +48,7 @@ INSTANCES = { "pattern": r"furbooru\.org", "filter_id": "2", }, -} - -BASE_PATTERN = PhilomenaExtractor.update(INSTANCES) +}) class PhilomenaPostExtractor(PhilomenaExtractor): @@ -176,10 +174,7 @@ class PhilomenaAPI(): if filter_id: params["filter_id"] = filter_id elif not api_key: - try: - params["filter_id"] = INSTANCES[extr.category]["filter_id"] - except (KeyError, TypeError): - params["filter_id"] = "2" + params["filter_id"] = extr.config_instance("filter_id") or "2" params["page"] = extr.page_start params["per_page"] = extr.per_page diff --git a/gallery_dl/extractor/shimmie2.py b/gallery_dl/extractor/shimmie2.py index 8a08fabb..3da8d42e 100644 --- a/gallery_dl/extractor/shimmie2.py +++ b/gallery_dl/extractor/shimmie2.py @@ -19,17 +19,12 @@ class Shimmie2Extractor(BaseExtractor): archive_fmt = "{id}" def _init(self): - try: - instance = INSTANCES[self.category] - except KeyError: - return - - cookies = instance.get("cookies") + cookies = self.config_instance("cookies") if cookies: domain = self.root.rpartition("/")[2] self.cookies_update_dict(cookies, domain=domain) - file_url = instance.get("file_url") + file_url = self.config_instance("file_url") if file_url: self.file_url_fmt = file_url @@ -73,7 +68,7 @@ class Shimmie2Extractor(BaseExtractor): return "'" -INSTANCES = { +BASE_PATTERN = Shimmie2Extractor.update({ "loudbooru": { "root": "https://loudbooru.com", "pattern": r"loudbooru\.com", @@ -97,9 +92,7 @@ INSTANCES = { "root": "https://rule34hentai.net", "pattern": r"rule34hentai\.net", }, -} - -BASE_PATTERN = Shimmie2Extractor.update(INSTANCES) + r"/(?:index\.php\?q=/?)?" +}) + r"/(?:index\.php\?q=/?)?" class Shimmie2TagExtractor(Shimmie2Extractor): diff --git a/gallery_dl/extractor/urlshortener.py b/gallery_dl/extractor/urlshortener.py index f2e65214..49a3debd 100644 --- a/gallery_dl/extractor/urlshortener.py +++ b/gallery_dl/extractor/urlshortener.py @@ -15,7 +15,7 @@ class UrlshortenerExtractor(BaseExtractor): basecategory = "urlshortener" -INSTANCES = { +BASE_PATTERN = UrlshortenerExtractor.update({ "bitly": { "root": "https://bit.ly", "pattern": r"bit\.ly", @@ -26,9 +26,7 @@ INSTANCES = { "root": "https://t.co", "pattern": r"t\.co", }, -} - -BASE_PATTERN = UrlshortenerExtractor.update(INSTANCES) +}) class UrlshortenerLinkExtractor(UrlshortenerExtractor): @@ -42,10 +40,7 @@ class UrlshortenerLinkExtractor(UrlshortenerExtractor): self.id = match.group(match.lastindex) def _init(self): - try: - self.headers = INSTANCES[self.category]["headers"] - except Exception: - self.headers = None + self.headers = self.config_instance("headers") def items(self): response = self.request( diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 34566465..859cbbad 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -457,7 +457,7 @@ def build_extractor_list(): domains[category] = domain(extr) else: base = categories[extr.basecategory] - for category, root in extr.instances: + for category, root, info in extr.instances: base[category].append(extr.subcategory) if category not in domains: if not root: