[luscious] add login capabilities (closes #159)

README.rst

@@ -154,9 +154,9 @@ Username & Password
 
 Some extractors require you to provide valid login-credentials in the form of
 a username & password pair.
-This is necessary for ``pixiv``, ``nijie`` and ``seiga`` and optional
-(but strongly recommended) for ``exhentai``,  ``sankaku``, ``idolcomplex``
-and ``wallhaven``.
+This is necessary for ``pixiv``, ``nijie`` and ``seiga``
+and optional (but strongly recommended) for ``exhentai``,  ``luscious``,
+``sankaku``, ``idolcomplex`` and ``wallhaven``.
 
 You can set the necessary information in your configuration file
 (cf. gallery-dl.conf_)

docs/supportedsites.rst

@@ -12,7 +12,7 @@ arch.b4k.co          https://arch.b4k.co/                Threads
 Archive of Sins      https://archiveofsins.com/          Threads
 Archived.Moe         https://archived.moe/               Threads
 ArtStation           https://www.artstation.com/         |Capabilities-0|
-Behance              https://www.behance.net/            Images from Users, Galleries
+Behance              https://www.behance.net/            Images from Users, Collections, Galleries
 BobX                 http://www.bobx.com/dark/           Galleries, Idols
 Danbooru             https://danbooru.donmai.us/         Pools, Popular Images, Posts, Tag-Searches         Optional
 Desuarchive          https://desuarchive.org/            Threads
@@ -47,7 +47,7 @@ Kirei Cake           https://reader.kireicake.com/       Chapters, Manga
 KissManga            http://kissmanga.com/               Chapters, Manga
 Komikcast            https://komikcast.com/              Chapters, Manga
 Konachan             https://konachan.com/               Pools, Popular Images, Posts, Tag-Searches
-Luscious             https://luscious.net/               Albums, Search Results
+Luscious             https://luscious.net/               Albums, Search Results                             Optional
 Manga Fox            http://fanfox.net/                  Chapters
 Manga Here           http://www.mangahere.co/            Chapters, Manga
 Manga Stream         https://mangastream.com/            Chapters
@@ -61,7 +61,8 @@ nhentai              https://nhentai.net/                Galleries, Search Resul
 Niconico Seiga       http://seiga.nicovideo.jp           Images from Users, individual Images               Required
 nijie                https://nijie.info/                 |Capabilities-5|                                   Required
 Nyafuu Archive       https://archive.nyafuu.org/         Threads
-Pawoo                https://pawoo.net                   Images from Users, Images from Statuses
+Pawoo                https://pawoo.net/                  Images from Users, Images from Statuses
+Photobucket          http://photobucket.com/             Albums, individual Images
 Piczel               https://piczel.tv/                  Images from Users, Folders, individual Images
 Pinterest            https://www.pinterest.com           Boards, Pins, pin.it Links, related Pins
 Pixiv                https://www.pixiv.net/              |Capabilities-6|                                   Required

gallery_dl/extractor/luscious.py

@@ -1,20 +1,54 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2016-2018 Mike Fährmann
+# Copyright 2016-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extract images from https://luscious.net/"""
+"""Extractors for https://luscious.net/"""
 
-from .common import Extractor, AsynchronousExtractor, Message
-from .. import text, util
+from .common import Extractor, Message
+from .. import text, util, exception
+from ..cache import cache
 
 
-class LusciousAlbumExtractor(AsynchronousExtractor):
-    """Extractor for image albums from luscious.net"""
+class LusciousExtractor(Extractor):
+    """Base class for luscious extractors"""
     category = "luscious"
+    cookiedomain = ".luscious.net"
+    root = "https://members.luscious.net"
+
+    def login(self):
+        """Login and set necessary cookies"""
+        username, password = self._get_auth_info()
+        if username:
+            cookie = self._login_impl(username, password)
+            self.session.cookies.set_cookie(cookie)
+
+    @cache(maxage=13*24*60*60, keyarg=1)
+    def _login_impl(self, username, password):
+        self.log.info("Logging in as %s", username)
+        url = "https://members.luscious.net/accounts/login/"
+        headers = {"Referer": "https://members.luscious.net/login/"}
+        data = {
+            "login": username,
+            "password": password,
+            "remember": "on",
+            "next": ""  "/",
+        }
+        response = self.request(url, method="POST", headers=headers, data=data)
+
+        if "/accounts/login/" in response.url or not response.history:
+            raise exception.AuthenticationError()
+        for cookie in response.history[0].cookies:
+            if cookie.name.startswith("sessionid_"):
+                return cookie
+        raise exception.AuthenticationError()
+
+
+class LusciousAlbumExtractor(LusciousExtractor):
+    """Extractor for image albums from luscious.net"""
     subcategory = "album"
     directory_fmt = ["{category}", "{gallery_id} {title}"]
     filename_fmt = "{category}_{gallery_id}_{num:>03}.{extension}"
@@ -36,13 +70,13 @@ class LusciousAlbumExtractor(AsynchronousExtractor):
         ("https://luscious.net/pictures/c/video_game_manga/album"
          "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", None),
     ]
-    root = "https://luscious.net"
 
     def __init__(self, match):
-        AsynchronousExtractor.__init__(self)
+        LusciousExtractor.__init__(self)
         self.gpart, self.gid = match.groups()
 
     def items(self):
+        self.login()
         url = "{}/albums/{}/".format(self.root, self.gpart)
         page = self.request(url).text
         data = self.get_metadata(page)
@@ -111,9 +145,8 @@ class LusciousAlbumExtractor(AsynchronousExtractor):
             num += 1
 
 
-class LusciousSearchExtractor(Extractor):
+class LusciousSearchExtractor(LusciousExtractor):
     """Extractor for album searches on luscious.net"""
-    category = "luscious"
     subcategory = "search"
     pattern = [(r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
                 r"/((?:albums|c)(?:/(?![^/?&#]+_\d+)[^/?&#]+)+)")]
@@ -126,15 +159,15 @@ class LusciousSearchExtractor(Extractor):
              "count": 21,
          }),
     ]
-    root = "https://luscious.net"
 
     def __init__(self, match):
-        Extractor.__init__(self)
+        LusciousExtractor.__init__(self)
         self.path = match.group(1).partition("/page/")[0]
         if not self.path.startswith("albums/"):
             self.path = "albums/" + self.path
 
     def items(self):
+        self.login()
         yield Message.Version, 1
         for album in self.albums():
             url, data = self.parse_album(album)

scripts/build_supportedsites.py

@@ -92,6 +92,7 @@ AUTH_MAP = {
     "exhentai"   : "Optional",
     "flickr"     : "Optional (OAuth)",
     "idolcomplex": "Optional",
+    "luscious"   : "Optional",
     "nijie"      : "Required",
     "pixiv"      : "Required",
     "reddit"     : "Optional (OAuth)",
@@ -209,7 +210,7 @@ def get_domain(classes):
             url = cls.test[0][0]
             return url[:url.find("/", 8)+1]
 
-        scheme = "https" if hasattr(cls, "https") and cls.https else "http"
+        scheme = "http" if hasattr(cls, "https") and not cls.https else "https"
         host = cls.__doc__.split()[-1]
         return scheme + "://" + host + "/"
     except (IndexError, AttributeError):