From 3ece3976ae401854be106c90a849d9795e262259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 15 Nov 2019 23:54:07 +0100 Subject: [PATCH] [newgrounds] implement login support (#394) --- docs/supportedsites.rst | 2 +- gallery_dl/extractor/newgrounds.py | 40 ++++++++++++++++++++++++++++-- scripts/supportedsites.py | 1 + test/test_results.py | 9 ++++--- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 4d2eddda..b239c7ed 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -75,7 +75,7 @@ MangaPark https://mangapark.me/ Chapters, Manga Mangareader https://www.mangareader.net/ Chapters, Manga Mangoxo https://www.mangoxo.com/ Albums, Channels Optional Naver https://blog.naver.com/ Blogs, Posts -Newgrounds https://www.newgrounds.com/ individual Images, User Profiles, Videos +Newgrounds https://www.newgrounds.com/ individual Images, User Profiles, Videos Optional Ngomik http://ngomik.in/ Chapters nhentai https://nhentai.net/ Galleries, Search Results Niconico Seiga https://seiga.nicovideo.jp/ individual Images, User Profiles Required diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 45109a67..dce6fb39 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -9,7 +9,8 @@ """Extractors for https://www.newgrounds.com/""" from .common import Extractor, Message -from .. import text +from .. import text, exception +from ..cache import cache import json @@ -19,13 +20,17 @@ class NewgroundsExtractor(Extractor): directory_fmt = ("{category}", "{user}") filename_fmt = "{category}_{index}_{title}.{extension}" archive_fmt = "{index}" + root = "https://www.newgrounds.com" + cookiedomain = ".newgrounds.com" + cookienames = ("NG_GG_username", "vmk1du5I8m") def __init__(self, match): Extractor.__init__(self, match) self.user = match.group(1) - self.root = "https://{}.newgrounds.com".format(self.user) + self.user_root = "https://{}.newgrounds.com".format(self.user) def items(self): + self.login() data = self.metadata() yield Message.Version, 1 @@ -66,6 +71,37 @@ class NewgroundsExtractor(Extractor): data["url"].rpartition("/")[2].partition("_")[0]) return data + def login(self): + username, password = self._get_auth_info() + if username: + self._update_cookies(self._login_impl(username, password)) + + @cache(maxage=360*24*3600, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + + url = self.root + "/passport/" + page = self.request(url).text + headers = {"Origin": self.root, "Referer": url} + + url = text.urljoin(self.root, text.extract(page, 'action="', '"')[0]) + data = { + "username": username, + "password": password, + "remember": "1", + "login" : "1", + } + + response = self.request(url, method="POST", headers=headers, data=data) + if not response.history: + raise exception.AuthenticationError() + + return { + cookie.name: cookie.value + for cookie in response.history[0].cookies + if cookie.expires and cookie.domain == self.cookiedomain + } + def _pagination(self, url): headers = { "Referer": self.root, diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 9b7a96d1..5ad5c14b 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -119,6 +119,7 @@ AUTH_MAP = { "imgbb" : "Optional", "instagram" : "Optional", "mangoxo" : "Optional", + "newgrounds" : "Optional", "nijie" : "Required", "pixiv" : "Required", "reddit" : "Optional (OAuth)", diff --git a/test/test_results.py b/test/test_results.py index b835fecf..8c797245 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -294,10 +294,11 @@ def setup_test_config(): config.set(("extractor", "nijie" , "username"), email) config.set(("extractor", "seiga" , "username"), email) - config.set(("extractor", "danbooru" , "username"), None) - config.set(("extractor", "instagram", "username"), None) - config.set(("extractor", "imgur" , "username"), None) - config.set(("extractor", "twitter" , "username"), None) + config.set(("extractor", "danbooru" , "username"), None) + config.set(("extractor", "instagram" , "username"), None) + config.set(("extractor", "imgur" , "username"), None) + config.set(("extractor", "newgrounds", "username"), None) + config.set(("extractor", "twitter" , "username"), None) config.set(("extractor", "mangoxo" , "username"), "LiQiang3") config.set(("extractor", "mangoxo" , "password"), "5zbQF10_5u25259Ma")