From b33efc99a496fc24af30749a659ea991c3284bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 9 Jan 2018 17:52:12 +0100 Subject: [PATCH] [idolcomplex] add support for idol.sankakucomplex.com --- CHANGELOG.md | 3 ++ docs/supportedsites.rst | 1 + gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/deviantart.py | 2 +- gallery_dl/extractor/idolcomplex.py | 48 +++++++++++++++++++++++++++++ gallery_dl/extractor/sankaku.py | 13 +++++--- scripts/build_supportedsites.py | 20 ++++++------ test/test_extractors.py | 4 ++- 8 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 gallery_dl/extractor/idolcomplex.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f989fea..7faecfe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## Unreleased +- Added support for: + - `puremashiro` - http://reader.puremashiro.moe/ ([#66](https://github.com/mikf/gallery-dl/issues/66)) + - `idolcomplex` - https://idol.sankakucomplex.com/ - Added an option to filter reblogs on `tumblr` ([#61](https://github.com/mikf/gallery-dl/issues/61)) - Improved pagination for various …booru sites to work around page limits - Fixed chapter information parsing for certain manga on `kissmanga` ([#58](https://github.com/mikf/gallery-dl/issues/58)) and `batoto` ([#60](https://github.com/mikf/gallery-dl/issues/60)) diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index c3206084..ea3e1b47 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -30,6 +30,7 @@ Hentai Foundry https://www.hentai-foundry.com/ Images from Users, indi Hentai2Read https://hentai2read.com/ Chapters, Manga HentaiHere https://hentaihere.com/ Chapters, Manga Hitomi.la https://hitomi.la/ Galleries +Idol Complex https://idol.sankakucomplex.com/ Pools, Posts, Tag-Searches Optional ImageBam http://www.imagebam.com/ Galleries, individual Images ImageFap http://imagefap.com/ Images from Users, Galleries, individual Images imgbox https://imgbox.com/ Galleries, individual Images diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index ee7d5710..784313a2 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -37,6 +37,7 @@ modules = [ "hentaifoundry", "hentaihere", "hitomi", + "idolcomplex", "imagebam", "imagefap", "imgbox", diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 903cc4b7..01d914a3 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -299,7 +299,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor): test = [(("https://pencilshadings.deviantart.com" "/favourites/70595441/3D-Favorites"), { "url": "742f92199d5bc6a89cda6ec6133d46c7a523824d", - "keyword": "22aa30e064cf8cd8b9ca7944f4877f5ac9793cfa", + "keyword": "9210c976b5274eff6ea1d2b8a4f891c9f35ce340", "options": (("original", False),), })] diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py new file mode 100644 index 00000000..0ed80d98 --- /dev/null +++ b/gallery_dl/extractor/idolcomplex.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extract images from https://idol.sankakucomplex.com/""" + +from . import sankaku + + +class IdolcomplexExtractor(sankaku.SankakuExtractor): + """Base class for idolcomplex extractors""" + category = "idolcomplex" + subdomain = "idol" + + +class IdolcomplexTagExtractor(IdolcomplexExtractor, + sankaku.SankakuTagExtractor): + """Extractor for images from idol.sankakucomplex.com by search-tags""" + pattern = [r"(?:https?://)?idol\.sankakucomplex\.com" + r"/\?(?:[^&#]*&)*tags=([^&#]+)"] + test = [("https://idol.sankakucomplex.com/?tags=lyumos+wreath", { + "count": ">= 6", + "pattern": (r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}" + r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+"), + })] + + +class IdolcomplexPoolExtractor(IdolcomplexExtractor, + sankaku.SankakuPoolExtractor): + """Extractor for image-pools from idol.sankakucomplex.com""" + pattern = [r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)"] + test = [("https://idol.sankakucomplex.com/pool/show/145", { + "count": 3, + })] + + +class IdolcomplexPostExtractor(IdolcomplexExtractor, + sankaku.SankakuPostExtractor): + """Extractor for single images from idol.sankakucomplex.com""" + pattern = [r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)"] + test = [("https://idol.sankakucomplex.com/post/show/694215", { + "content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + "count": 1, + })] diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index 4aec5313..eddf2a16 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -20,12 +20,14 @@ class SankakuExtractor(SharedConfigExtractor): basecategory = "booru" category = "sankaku" filename_fmt = "{category}_{id}_{md5}.{extension}" - root = "https://chan.sankakucomplex.com" cookienames = ("login", "pass_hash") - cookiedomain = "chan.sankakucomplex.com" + subdomain = "chan" def __init__(self): SharedConfigExtractor.__init__(self) + self.cookiedomain = self.subdomain + ".sankakucomplex.com" + self.root = "https://" + self.cookiedomain + self.logged_in = True self.start_page = 1 self.start_post = 0 @@ -61,7 +63,7 @@ class SankakuExtractor(SharedConfigExtractor): url = self.root + "/post/show/" + post_id page = self.request(url, retries=10).text - tags , pos = extr(page, "", " | Sankaku Channel") + tags , pos = extr(page, "", " | ") vavg , pos = extr(page, "itemprop=ratingValue>", "<", pos) vcnt , pos = extr(page, "itemprop=reviewCount>", "<", pos) _ , pos = extr(page, "Posted: <", "", pos) @@ -100,7 +102,7 @@ class SankakuExtractor(SharedConfigExtractor): return username, password = self._get_auth_info() if username: - cookies = self._login_impl(username, password) + cookies = self._login_impl((username, self.subdomain), password) for key, value in cookies.items(): self.session.cookies.set( key, value, domain=self.cookiedomain) @@ -108,8 +110,9 @@ class SankakuExtractor(SharedConfigExtractor): self.logged_in = False @cache(maxage=90*24*60*60, keyarg=1) - def _login_impl(self, username, password): + def _login_impl(self, usertuple, password): """Actual login implementation""" + username = usertuple[0] self.log.info("Logging in as %s", username) params = { "url": "", diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index 0d703d9b..7d6cd429 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -25,6 +25,7 @@ CATEGORY_MAP = { "hentaifoundry" : "Hentai Foundry", "hentaihere" : "HentaiHere", "hitomi" : "Hitomi.la", + "idolcomplex" : "Idol Complex", "imagebam" : "ImageBam", "imagefap" : "ImageFap", "imgbox" : "imgbox", @@ -77,15 +78,16 @@ SUBCATEGORY_MAP = { } AUTH_MAP = { - "batoto" : "Optional", - "deviantart": "Optional (OAuth)", - "exhentai" : "Optional", - "flickr" : "Optional (OAuth)", - "nijie" : "Required", - "pixiv" : "Required", - "reddit" : "Optional (OAuth)", - "sankaku" : "Optional", - "seiga" : "Required", + "batoto" : "Optional", + "deviantart" : "Optional (OAuth)", + "exhentai" : "Optional", + "flickr" : "Optional (OAuth)", + "idolcomplex": "Optional", + "nijie" : "Required", + "pixiv" : "Required", + "reddit" : "Optional (OAuth)", + "sankaku" : "Optional", + "seiga" : "Required", } IGNORE_LIST = ( diff --git a/test/test_extractors.py b/test/test_extractors.py index 9a2110f6..f35a8d84 100644 --- a/test/test_extractors.py +++ b/test/test_extractors.py @@ -18,7 +18,9 @@ SKIP = { "archivedmoe", "archiveofsins", "thebarchive", # temporary issues - + "batoto", # R.I.P. + "imgyt", # server maintenance + "loveisover", }