gallery-dl/gallery_dl/extractor/twitter.py

# -*- coding: utf-8 -*-

# Copyright 2016 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Extract images from https://twitter.com/"""

from .common import Extractor, Message
from .. import text

class TwitterTweetExtractor(Extractor):
    """Extractor for images from twitter tweets"""
    category = "twitter"
    subcategory = "tweet"
    directory_fmt = ["{category}", "{user}"]
    filename_fmt = "{tweet-id}_{num}.{extension}"
    pattern = [r"(?:https?://)?(?:www\.|mobile\.)?twitter\.com/(([^/]+)/status/(\d+))"]
    test = [("https://twitter.com/PicturesEarth/status/672897688871018500", {
        "url": "d9e68d41301d2fe382eb27711dea28366be03b1a",
        "keyword": "3cd8e27026a2112008985b1b53f5e4baf4616177",
        "content": "a1f2f04cb2d8df24b1afa7a39910afda23484342",
    })]

    def __init__(self, match):
        Extractor.__init__(self)
        self.path, self.user, self.tid = match.groups()

    def items(self):
        self.session.headers["User-Agent"] = (
            "Mozilla/5.0 (X11; Linux x86_64; rv:48.0) "
            "Gecko/20100101 Firefox/48.0"
        )
        page = self.request("https://mobile.twitter.com/" + self.path).text
        data = self.get_job_metadata()
        imgs = self.get_image_urls(page)
        data["count"] = len(imgs)
        yield Message.Version, 1
        yield Message.Directory, data
        for data["num"], url in enumerate(imgs, 1):
            yield Message.Url, url + ":orig", text.nameext_from_url(url, data)

    def get_job_metadata(self):
        """Collect metadata for extractor-job"""
        return {
            "user": self.user,
            "tweet-id": self.tid,
        }

    @staticmethod
    def get_image_urls(page):
        """Extract and return a list of all image-urls"""
        return list(text.extract_iter(page, 'alt="Embedded image" src="', '"'))