diff --git a/README.rst b/README.rst index fb3a07a7..15ac73c2 100644 --- a/README.rst +++ b/README.rst @@ -154,9 +154,9 @@ Username & Password Some extractors require you to provide valid login-credentials in the form of a username & password pair. -This is necessary for ``pixiv``, ``nijie`` and ``seiga`` and optional -(but strongly recommended) for ``exhentai``, ``sankaku``, ``idolcomplex`` -and ``wallhaven``. +This is necessary for ``pixiv``, ``nijie`` and ``seiga`` +and optional (but strongly recommended) for ``exhentai``, ``luscious``, +``sankaku``, ``idolcomplex`` and ``wallhaven``. You can set the necessary information in your configuration file (cf. gallery-dl.conf_) diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 2316e820..8ec4da2f 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -12,7 +12,7 @@ arch.b4k.co https://arch.b4k.co/ Threads Archive of Sins https://archiveofsins.com/ Threads Archived.Moe https://archived.moe/ Threads ArtStation https://www.artstation.com/ |Capabilities-0| -Behance https://www.behance.net/ Images from Users, Galleries +Behance https://www.behance.net/ Images from Users, Collections, Galleries BobX http://www.bobx.com/dark/ Galleries, Idols Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches Optional Desuarchive https://desuarchive.org/ Threads @@ -47,7 +47,7 @@ Kirei Cake https://reader.kireicake.com/ Chapters, Manga KissManga http://kissmanga.com/ Chapters, Manga Komikcast https://komikcast.com/ Chapters, Manga Konachan https://konachan.com/ Pools, Popular Images, Posts, Tag-Searches -Luscious https://luscious.net/ Albums, Search Results +Luscious https://luscious.net/ Albums, Search Results Optional Manga Fox http://fanfox.net/ Chapters Manga Here http://www.mangahere.co/ Chapters, Manga Manga Stream https://mangastream.com/ Chapters @@ -61,7 +61,8 @@ nhentai https://nhentai.net/ Galleries, Search Resul Niconico Seiga http://seiga.nicovideo.jp Images from Users, individual Images Required nijie https://nijie.info/ |Capabilities-5| Required Nyafuu Archive https://archive.nyafuu.org/ Threads -Pawoo https://pawoo.net Images from Users, Images from Statuses +Pawoo https://pawoo.net/ Images from Users, Images from Statuses +Photobucket http://photobucket.com/ Albums, individual Images Piczel https://piczel.tv/ Images from Users, Folders, individual Images Pinterest https://www.pinterest.com Boards, Pins, pin.it Links, related Pins Pixiv https://www.pixiv.net/ |Capabilities-6| Required diff --git a/gallery_dl/extractor/luscious.py b/gallery_dl/extractor/luscious.py index af06d4cd..84ba5757 100644 --- a/gallery_dl/extractor/luscious.py +++ b/gallery_dl/extractor/luscious.py @@ -1,20 +1,54 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2018 Mike Fährmann +# Copyright 2016-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://luscious.net/""" +"""Extractors for https://luscious.net/""" -from .common import Extractor, AsynchronousExtractor, Message -from .. import text, util +from .common import Extractor, Message +from .. import text, util, exception +from ..cache import cache -class LusciousAlbumExtractor(AsynchronousExtractor): - """Extractor for image albums from luscious.net""" +class LusciousExtractor(Extractor): + """Base class for luscious extractors""" category = "luscious" + cookiedomain = ".luscious.net" + root = "https://members.luscious.net" + + def login(self): + """Login and set necessary cookies""" + username, password = self._get_auth_info() + if username: + cookie = self._login_impl(username, password) + self.session.cookies.set_cookie(cookie) + + @cache(maxage=13*24*60*60, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + url = "https://members.luscious.net/accounts/login/" + headers = {"Referer": "https://members.luscious.net/login/"} + data = { + "login": username, + "password": password, + "remember": "on", + "next": "" "/", + } + response = self.request(url, method="POST", headers=headers, data=data) + + if "/accounts/login/" in response.url or not response.history: + raise exception.AuthenticationError() + for cookie in response.history[0].cookies: + if cookie.name.startswith("sessionid_"): + return cookie + raise exception.AuthenticationError() + + +class LusciousAlbumExtractor(LusciousExtractor): + """Extractor for image albums from luscious.net""" subcategory = "album" directory_fmt = ["{category}", "{gallery_id} {title}"] filename_fmt = "{category}_{gallery_id}_{num:>03}.{extension}" @@ -36,13 +70,13 @@ class LusciousAlbumExtractor(AsynchronousExtractor): ("https://luscious.net/pictures/c/video_game_manga/album" "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", None), ] - root = "https://luscious.net" def __init__(self, match): - AsynchronousExtractor.__init__(self) + LusciousExtractor.__init__(self) self.gpart, self.gid = match.groups() def items(self): + self.login() url = "{}/albums/{}/".format(self.root, self.gpart) page = self.request(url).text data = self.get_metadata(page) @@ -111,9 +145,8 @@ class LusciousAlbumExtractor(AsynchronousExtractor): num += 1 -class LusciousSearchExtractor(Extractor): +class LusciousSearchExtractor(LusciousExtractor): """Extractor for album searches on luscious.net""" - category = "luscious" subcategory = "search" pattern = [(r"(?:https?://)?(?:www\.|members\.)?luscious\.net" r"/((?:albums|c)(?:/(?![^/?&#]+_\d+)[^/?&#]+)+)")] @@ -126,15 +159,15 @@ class LusciousSearchExtractor(Extractor): "count": 21, }), ] - root = "https://luscious.net" def __init__(self, match): - Extractor.__init__(self) + LusciousExtractor.__init__(self) self.path = match.group(1).partition("/page/")[0] if not self.path.startswith("albums/"): self.path = "albums/" + self.path def items(self): + self.login() yield Message.Version, 1 for album in self.albums(): url, data = self.parse_album(album) diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index 6fd51528..13ffe4e3 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -92,6 +92,7 @@ AUTH_MAP = { "exhentai" : "Optional", "flickr" : "Optional (OAuth)", "idolcomplex": "Optional", + "luscious" : "Optional", "nijie" : "Required", "pixiv" : "Required", "reddit" : "Optional (OAuth)", @@ -209,7 +210,7 @@ def get_domain(classes): url = cls.test[0][0] return url[:url.find("/", 8)+1] - scheme = "https" if hasattr(cls, "https") and cls.https else "http" + scheme = "http" if hasattr(cls, "https") and not cls.https else "https" host = cls.__doc__.split()[-1] return scheme + "://" + host + "/" except (IndexError, AttributeError):