[twitter] add login support (#214)

docs/supportedsites.rst

@@ -91,7 +91,7 @@ SmugMug              https://www.smugmug.com/            |smugmug-C|
 The /b/ Archive      https://thebarchive.com/            Threads
 Tsumino              https://www.tsumino.com/            Galleries, Search Results                          Optional
 Tumblr               https://www.tumblr.com/             Images from Users, Likes, Posts, Tag-Searches      Optional (OAuth)
-Twitter              https://twitter.com/                Media Timelines, Timelines, Tweets
+Twitter              https://twitter.com/                Media Timelines, Timelines, Tweets                 Optional
 Wallhaven            https://alpha.wallhaven.cc/         individual Images, Search Results                  Optional
 Warosu               https://warosu.org/                 Threads
 Weibo                https://www.weibo.com/              Images from Users, Images from Statuses
@@ -99,7 +99,7 @@ WikiArt.org          https://www.wikiart.org/            Artists, Artworks
 World Three          http://www.slide.world-three.org/   Chapters, Manga
 XVideos              https://www.xvideos.com/            Images from Users, Galleries
 Yandere              https://yande.re/                   Pools, Popular Images, Posts, Tag-Searches
-yaplog!              https://yaplog.jp/                  Images from Users, Posts
+yaplog!              https://yaplog.jp/                  Blogs, Posts
 |yuki-S|             https://yuki.la/                    Threads
 Acidimg              https://acidimg.cc/                 individual Images
 Imagetwist           https://imagetwist.com/             individual Images

gallery_dl/extractor/twitter.py

@@ -9,7 +9,8 @@
 """Extract images from https://twitter.com/"""
 
 from .common import Extractor, Message
-from .. import text
+from .. import text, exception
+from ..cache import cache
 
 
 class TwitterExtractor(Extractor):
@@ -27,6 +28,7 @@ class TwitterExtractor(Extractor):
         self.videos = self.config("videos", False)
 
     def items(self):
+        self.login()
         yield Message.Version, 1
         yield Message.Directory, self.metadata()
 
@@ -54,6 +56,35 @@ class TwitterExtractor(Extractor):
     def tweets(self):
         """Yield HTML content of all relevant tweets"""
 
+    def login(self):
+        username, password = self._get_auth_info()
+        if username:
+            self._update_cookies(self._login_impl(username, password))
+
+    @cache(maxage=360*24*3600, keyarg=1)
+    def _login_impl(self, username, password):
+        self.log.info("Logging in as %s", username)
+
+        page = self.request(self.root + "/login").text
+        pos = page.index('name="authenticity_token"')
+        token = text.extract(page, 'value="', '"', pos-80)[0]
+
+        url = self.root + "/sessions"
+        data = {
+            "session[username_or_email]": username,
+            "session[password]"         : password,
+            "authenticity_token"        : token,
+            "ui_metrics"                : '{"rf":{},"s":""}',
+            "scribe_log"                : "",
+            "redirect_after_login"      : "",
+            "remember_me"               : "1",
+        }
+        response = self.request(url, method="POST", data=data)
+
+        if "/error" in response.url:
+            raise exception.AuthenticationError()
+        return self.session.cookies
+
     @staticmethod
     def _data_from_tweet(tweet):
         data = text.extract_all(tweet, (

scripts/supportedsites.py

@@ -110,6 +110,7 @@ AUTH_MAP = {
     "smugmug"    : "Optional (OAuth)",
     "tsumino"    : "Optional",
     "tumblr"     : "Optional (OAuth)",
+    "twitter"    : "Optional",
     "wallhaven"  : "Optional",
 }
 

test/test_results.py

@@ -227,6 +227,7 @@ def setup_test_config():
     config.set(("extractor", "nijie", "username"), email)
     config.set(("extractor", "seiga", "username"), email)
     config.set(("extractor", "danbooru", "username"), None)
+    config.set(("extractor", "twitter" , "username"), None)
 
     config.set(("extractor", "deviantart", "client-id"), "7777")
     config.set(("extractor", "deviantart", "client-secret"),