[newgrounds] implement login support (#394)

docs/supportedsites.rst

@@ -75,7 +75,7 @@ MangaPark            https://mangapark.me/               Chapters, Manga
 Mangareader          https://www.mangareader.net/        Chapters, Manga
 Mangoxo              https://www.mangoxo.com/            Albums, Channels                                   Optional
 Naver                https://blog.naver.com/             Blogs, Posts
-Newgrounds           https://www.newgrounds.com/         individual Images, User Profiles, Videos
+Newgrounds           https://www.newgrounds.com/         individual Images, User Profiles, Videos           Optional
 Ngomik               http://ngomik.in/                   Chapters
 nhentai              https://nhentai.net/                Galleries, Search Results
 Niconico Seiga       https://seiga.nicovideo.jp/         individual Images, User Profiles                   Required

gallery_dl/extractor/newgrounds.py

@@ -9,7 +9,8 @@
 """Extractors for https://www.newgrounds.com/"""
 
 from .common import Extractor, Message
-from .. import text
+from .. import text, exception
+from ..cache import cache
 import json
 
 
@@ -19,13 +20,17 @@ class NewgroundsExtractor(Extractor):
     directory_fmt = ("{category}", "{user}")
     filename_fmt = "{category}_{index}_{title}.{extension}"
     archive_fmt = "{index}"
+    root = "https://www.newgrounds.com"
+    cookiedomain = ".newgrounds.com"
+    cookienames = ("NG_GG_username", "vmk1du5I8m")
 
     def __init__(self, match):
         Extractor.__init__(self, match)
         self.user = match.group(1)
-        self.root = "https://{}.newgrounds.com".format(self.user)
+        self.user_root = "https://{}.newgrounds.com".format(self.user)
 
     def items(self):
+        self.login()
         data = self.metadata()
         yield Message.Version, 1
 
@@ -66,6 +71,37 @@ class NewgroundsExtractor(Extractor):
             data["url"].rpartition("/")[2].partition("_")[0])
         return data
 
+    def login(self):
+        username, password = self._get_auth_info()
+        if username:
+            self._update_cookies(self._login_impl(username, password))
+
+    @cache(maxage=360*24*3600, keyarg=1)
+    def _login_impl(self, username, password):
+        self.log.info("Logging in as %s", username)
+
+        url = self.root + "/passport/"
+        page = self.request(url).text
+        headers = {"Origin": self.root, "Referer": url}
+
+        url = text.urljoin(self.root, text.extract(page, 'action="', '"')[0])
+        data = {
+            "username": username,
+            "password": password,
+            "remember": "1",
+            "login"   : "1",
+        }
+
+        response = self.request(url, method="POST", headers=headers, data=data)
+        if not response.history:
+            raise exception.AuthenticationError()
+
+        return {
+            cookie.name: cookie.value
+            for cookie in response.history[0].cookies
+            if cookie.expires and cookie.domain == self.cookiedomain
+        }
+
     def _pagination(self, url):
         headers = {
             "Referer": self.root,

scripts/supportedsites.py

@@ -119,6 +119,7 @@ AUTH_MAP = {
     "imgbb"      : "Optional",
     "instagram"  : "Optional",
     "mangoxo"    : "Optional",
+    "newgrounds" : "Optional",
     "nijie"      : "Required",
     "pixiv"      : "Required",
     "reddit"     : "Optional (OAuth)",

test/test_results.py

@@ -294,10 +294,11 @@ def setup_test_config():
     config.set(("extractor", "nijie"    , "username"), email)
     config.set(("extractor", "seiga"    , "username"), email)
 
-    config.set(("extractor", "danbooru" , "username"), None)
+    config.set(("extractor", "danbooru"  , "username"), None)
-    config.set(("extractor", "instagram", "username"), None)
+    config.set(("extractor", "instagram" , "username"), None)
-    config.set(("extractor", "imgur"    , "username"), None)
+    config.set(("extractor", "imgur"     , "username"), None)
-    config.set(("extractor", "twitter"  , "username"), None)
+    config.set(("extractor", "newgrounds", "username"), None)
+    config.set(("extractor", "twitter"   , "username"), None)
 
     config.set(("extractor", "mangoxo"  , "username"), "LiQiang3")
     config.set(("extractor", "mangoxo"  , "password"), "5zbQF10_5u25259Ma")