|
|
|
@ -1,6 +1,6 @@
|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
|
|
# Copyright 2019-2020 Mike Fährmann
|
|
|
|
|
# Copyright 2019-2021 Mike Fährmann
|
|
|
|
|
#
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
@ -8,35 +8,25 @@
|
|
|
|
|
|
|
|
|
|
"""Extractors for mastodon instances"""
|
|
|
|
|
|
|
|
|
|
from .common import Extractor, Message
|
|
|
|
|
from .. import text, util, config, exception
|
|
|
|
|
import re
|
|
|
|
|
from .common import BaseExtractor, Message
|
|
|
|
|
from .. import text, exception
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MastodonExtractor(Extractor):
|
|
|
|
|
class MastodonExtractor(BaseExtractor):
|
|
|
|
|
"""Base class for mastodon extractors"""
|
|
|
|
|
basecategory = "mastodon"
|
|
|
|
|
directory_fmt = ("mastodon", "{instance}", "{account[username]}")
|
|
|
|
|
filename_fmt = "{category}_{id}_{media[id]}.{extension}"
|
|
|
|
|
archive_fmt = "{media[id]}"
|
|
|
|
|
cookiedomain = None
|
|
|
|
|
instance = None
|
|
|
|
|
root = None
|
|
|
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
|
Extractor.__init__(self, match)
|
|
|
|
|
BaseExtractor.__init__(self, match)
|
|
|
|
|
self.instance = self.root.partition("://")[2]
|
|
|
|
|
self.item = match.group(match.lastindex)
|
|
|
|
|
self.api = MastodonAPI(self)
|
|
|
|
|
|
|
|
|
|
def config(self, key, default=None):
|
|
|
|
|
return config.interpolate_common(
|
|
|
|
|
("extractor",), (
|
|
|
|
|
(self.category, self.subcategory),
|
|
|
|
|
(self.basecategory, self.instance, self.subcategory),
|
|
|
|
|
), key, default,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def items(self):
|
|
|
|
|
yield Message.Version, 1
|
|
|
|
|
for status in self.statuses():
|
|
|
|
|
attachments = status["media_attachments"]
|
|
|
|
|
if attachments:
|
|
|
|
@ -60,18 +50,56 @@ class MastodonExtractor(Extractor):
|
|
|
|
|
status["created_at"][:19], "%Y-%m-%dT%H:%M:%S")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
INSTANCES = {
|
|
|
|
|
"mastodon.social": {
|
|
|
|
|
"root" : "https://mastodon.social",
|
|
|
|
|
"access-token" : "Y06R36SMvuXXN5_wiPKFAEFiQaMSQg0o_hGgc86Jj48",
|
|
|
|
|
"client-id" : "dBSHdpsnOUZgxOnjKSQrWEPakO3ctM7HmsyoOd4FcRo",
|
|
|
|
|
"client-secret": "DdrODTHs_XoeOsNVXnILTMabtdpWrWOAtrmw91wU1zI",
|
|
|
|
|
},
|
|
|
|
|
"pawoo": {
|
|
|
|
|
"root" : "https://pawoo.net",
|
|
|
|
|
"access-token" : "c12c9d275050bce0dc92169a28db09d7"
|
|
|
|
|
"0d62d0a75a8525953098c167eacd3668",
|
|
|
|
|
"client-id" : "978a25f843ec01e53d09be2c290cd75c"
|
|
|
|
|
"782bc3b7fdbd7ea4164b9f3c3780c8ff",
|
|
|
|
|
"client-secret": "9208e3d4a7997032cf4f1b0e12e5df38"
|
|
|
|
|
"8428ef1fadb446dcfeb4f5ed6872d97b",
|
|
|
|
|
},
|
|
|
|
|
"baraag": {
|
|
|
|
|
"root" : "https://baraag.net",
|
|
|
|
|
"access-token" : "53P1Mdigf4EJMH-RmeFOOSM9gdSDztmrAYFgabOKKE0",
|
|
|
|
|
"client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o",
|
|
|
|
|
"client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BASE_PATTERN = MastodonExtractor.update(INSTANCES)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MastodonUserExtractor(MastodonExtractor):
|
|
|
|
|
"""Extractor for all images of an account/user"""
|
|
|
|
|
subcategory = "user"
|
|
|
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
|
MastodonExtractor.__init__(self, match)
|
|
|
|
|
self.account_name = match.group(1)
|
|
|
|
|
pattern = BASE_PATTERN + r"/@([^/?#]+)(?:/media)?/?$"
|
|
|
|
|
test = (
|
|
|
|
|
("https://mastodon.social/@jk", {
|
|
|
|
|
"pattern": r"https://files.mastodon.social/media_attachments"
|
|
|
|
|
r"/files/(\d+/){3,}original/\w+",
|
|
|
|
|
"range": "1-60",
|
|
|
|
|
"count": 60,
|
|
|
|
|
}),
|
|
|
|
|
("https://pawoo.net/@yoru_nine/", {
|
|
|
|
|
"range": "1-60",
|
|
|
|
|
"count": 60,
|
|
|
|
|
}),
|
|
|
|
|
("https://baraag.net/@pumpkinnsfw"),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def statuses(self):
|
|
|
|
|
handle = "@{}@{}".format(self.account_name, self.instance)
|
|
|
|
|
username = self.item
|
|
|
|
|
handle = "@{}@{}".format(username, self.instance)
|
|
|
|
|
for account in self.api.account_search(handle, 1):
|
|
|
|
|
if account["username"] == self.account_name:
|
|
|
|
|
if account["username"] == username:
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
raise exception.NotFoundError("account")
|
|
|
|
@ -81,13 +109,21 @@ class MastodonUserExtractor(MastodonExtractor):
|
|
|
|
|
class MastodonStatusExtractor(MastodonExtractor):
|
|
|
|
|
"""Extractor for images from a status"""
|
|
|
|
|
subcategory = "status"
|
|
|
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
|
MastodonExtractor.__init__(self, match)
|
|
|
|
|
self.status_id = match.group(1)
|
|
|
|
|
pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)"
|
|
|
|
|
test = (
|
|
|
|
|
("https://mastodon.social/@jk/103794036899778366", {
|
|
|
|
|
"count": 4,
|
|
|
|
|
}),
|
|
|
|
|
("https://pawoo.net/@yoru_nine/105038878897832922", {
|
|
|
|
|
"content": "b52e807f8ab548d6f896b09218ece01eba83987a",
|
|
|
|
|
}),
|
|
|
|
|
("https://baraag.net/@pumpkinnsfw/104364170556898443", {
|
|
|
|
|
"content": "67748c1b828c58ad60d0fe5729b59fb29c872244",
|
|
|
|
|
}),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def statuses(self):
|
|
|
|
|
return (self.api.status(self.status_id),)
|
|
|
|
|
return (self.api.status(self.item),)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MastodonAPI():
|
|
|
|
@ -102,30 +138,36 @@ class MastodonAPI():
|
|
|
|
|
self.extractor = extractor
|
|
|
|
|
|
|
|
|
|
if not access_token:
|
|
|
|
|
access_token = extractor.config(
|
|
|
|
|
"access-token", extractor.access_token)
|
|
|
|
|
self.headers = {"Authorization": "Bearer {}".format(access_token)}
|
|
|
|
|
access_token = extractor.config("access-token")
|
|
|
|
|
if not access_token:
|
|
|
|
|
if extractor.category not in INSTANCES:
|
|
|
|
|
raise exception.StopExtraction("missing access token")
|
|
|
|
|
access_token = INSTANCES[extractor.category]["access-token"]
|
|
|
|
|
|
|
|
|
|
self.headers = {"Authorization": "Bearer " + access_token}
|
|
|
|
|
|
|
|
|
|
def account_search(self, query, limit=40):
|
|
|
|
|
"""Search for content"""
|
|
|
|
|
endpoint = "/v1/accounts/search"
|
|
|
|
|
params = {"q": query, "limit": limit}
|
|
|
|
|
return self._call("accounts/search", params).json()
|
|
|
|
|
return self._call(endpoint, params).json()
|
|
|
|
|
|
|
|
|
|
def account_statuses(self, account_id):
|
|
|
|
|
"""Get an account's statuses"""
|
|
|
|
|
endpoint = "accounts/{}/statuses".format(account_id)
|
|
|
|
|
endpoint = "/v1/accounts/{}/statuses".format(account_id)
|
|
|
|
|
params = {"only_media": "1"}
|
|
|
|
|
return self._pagination(endpoint, params)
|
|
|
|
|
|
|
|
|
|
def status(self, status_id):
|
|
|
|
|
"""Fetch a Status"""
|
|
|
|
|
return self._call("statuses/" + status_id).json()
|
|
|
|
|
"""Fetch a status"""
|
|
|
|
|
endpoint = "/v1/statuses/" + status_id
|
|
|
|
|
return self._call(endpoint).json()
|
|
|
|
|
|
|
|
|
|
def _call(self, endpoint, params=None):
|
|
|
|
|
if endpoint.startswith("http"):
|
|
|
|
|
url = endpoint
|
|
|
|
|
else:
|
|
|
|
|
url = "{}/api/v1/{}".format(self.root, endpoint)
|
|
|
|
|
url = self.root + "/api" + endpoint
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
|
response = self.extractor.request(
|
|
|
|
@ -145,7 +187,7 @@ class MastodonAPI():
|
|
|
|
|
raise exception.StopExtraction(response.json().get("error"))
|
|
|
|
|
|
|
|
|
|
def _pagination(self, endpoint, params):
|
|
|
|
|
url = "{}/api/v1/{}".format(self.root, endpoint)
|
|
|
|
|
url = endpoint
|
|
|
|
|
while url:
|
|
|
|
|
response = self._call(url, params)
|
|
|
|
|
yield from response.json()
|
|
|
|
@ -154,88 +196,3 @@ class MastodonAPI():
|
|
|
|
|
if not url:
|
|
|
|
|
return
|
|
|
|
|
url = url["url"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def generate_extractors():
|
|
|
|
|
"""Dynamically generate Extractor classes for Mastodon instances"""
|
|
|
|
|
|
|
|
|
|
symtable = globals()
|
|
|
|
|
extractors = config.get(("extractor",), "mastodon")
|
|
|
|
|
if extractors:
|
|
|
|
|
util.combine_dict(EXTRACTORS, extractors)
|
|
|
|
|
config.set(("extractor",), "mastodon", EXTRACTORS)
|
|
|
|
|
|
|
|
|
|
for instance, info in EXTRACTORS.items():
|
|
|
|
|
|
|
|
|
|
if not isinstance(info, dict):
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
category = info.get("category") or instance.replace(".", "")
|
|
|
|
|
root = info.get("root") or "https://" + instance
|
|
|
|
|
name = (info.get("name") or category).capitalize()
|
|
|
|
|
token = info.get("access-token")
|
|
|
|
|
pattern = info.get("pattern") or re.escape(instance)
|
|
|
|
|
|
|
|
|
|
class Extr(MastodonUserExtractor):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
Extr.__name__ = Extr.__qualname__ = name + "UserExtractor"
|
|
|
|
|
Extr.__doc__ = "Extractor for all images of a user on " + instance
|
|
|
|
|
Extr.category = category
|
|
|
|
|
Extr.instance = instance
|
|
|
|
|
Extr.pattern = (r"(?:https?://)?" + pattern +
|
|
|
|
|
r"/@([^/?#]+)(?:/media)?/?$")
|
|
|
|
|
Extr.test = info.get("test-user")
|
|
|
|
|
Extr.root = root
|
|
|
|
|
Extr.access_token = token
|
|
|
|
|
symtable[Extr.__name__] = Extr
|
|
|
|
|
|
|
|
|
|
class Extr(MastodonStatusExtractor):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
Extr.__name__ = Extr.__qualname__ = name + "StatusExtractor"
|
|
|
|
|
Extr.__doc__ = "Extractor for images from a status on " + instance
|
|
|
|
|
Extr.category = category
|
|
|
|
|
Extr.instance = instance
|
|
|
|
|
Extr.pattern = r"(?:https?://)?" + pattern + r"/@[^/?#]+/(\d+)"
|
|
|
|
|
Extr.test = info.get("test-status")
|
|
|
|
|
Extr.root = root
|
|
|
|
|
Extr.access_token = token
|
|
|
|
|
symtable[Extr.__name__] = Extr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
EXTRACTORS = {
|
|
|
|
|
"mastodon.social": {
|
|
|
|
|
"category" : "mastodon.social",
|
|
|
|
|
"access-token" : "Y06R36SMvuXXN5_wiPKFAEFiQaMSQg0o_hGgc86Jj48",
|
|
|
|
|
"client-id" : "dBSHdpsnOUZgxOnjKSQrWEPakO3ctM7HmsyoOd4FcRo",
|
|
|
|
|
"client-secret": "DdrODTHs_XoeOsNVXnILTMabtdpWrWOAtrmw91wU1zI",
|
|
|
|
|
"test-user" : ("https://mastodon.social/@jk", {
|
|
|
|
|
"pattern": r"https://files.mastodon.social/media_attachments"
|
|
|
|
|
r"/files/(\d+/){3,}original/\w+",
|
|
|
|
|
"range": "1-60",
|
|
|
|
|
"count": 60,
|
|
|
|
|
}),
|
|
|
|
|
"test-status" : ("https://mastodon.social/@jk/103794036899778366", {
|
|
|
|
|
"count": 4,
|
|
|
|
|
}),
|
|
|
|
|
},
|
|
|
|
|
"pawoo.net": {
|
|
|
|
|
"category" : "pawoo",
|
|
|
|
|
"access-token" : "c12c9d275050bce0dc92169a28db09d7"
|
|
|
|
|
"0d62d0a75a8525953098c167eacd3668",
|
|
|
|
|
"client-id" : "978a25f843ec01e53d09be2c290cd75c"
|
|
|
|
|
"782bc3b7fdbd7ea4164b9f3c3780c8ff",
|
|
|
|
|
"client-secret": "9208e3d4a7997032cf4f1b0e12e5df38"
|
|
|
|
|
"8428ef1fadb446dcfeb4f5ed6872d97b",
|
|
|
|
|
},
|
|
|
|
|
"baraag.net": {
|
|
|
|
|
"category" : "baraag",
|
|
|
|
|
"access-token" : "53P1Mdigf4EJMH-RmeFOOSM9gdSDztmrAYFgabOKKE0",
|
|
|
|
|
"client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o",
|
|
|
|
|
"client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY",
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
generate_extractors()
|
|
|
|
|