From 04dbfd994e34163bca922453e0aee47eb85886ca Mon Sep 17 00:00:00 2001 From: Naatie Date: Sun, 23 Apr 2023 15:24:25 +0700 Subject: [PATCH 1/3] [misskey] add my favorites extractor --- gallery_dl/extractor/misskey.py | 38 ++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 03e91045..7e5ef799 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -7,7 +7,7 @@ """Extractors for Misskey instances""" from .common import BaseExtractor, Message -from .. import text +from .. import text, exception class MisskeyExtractor(BaseExtractor): @@ -152,6 +152,33 @@ class MisskeyNoteExtractor(MisskeyExtractor): return (self.api.notes_show(self.item),) +class MisskeyMyFavoritesExtractor(MisskeyExtractor): + """Extractor for images from favorites""" + subcategory = "favorites" + pattern = BASE_PATTERN + r"(/my/favorites|/api/i/favorites)" + test = ( + ("https://misskey.io/my/favorites",), + ("https://misskey.io/api/i/favorites",), + ) + + def items(self): + for fav in self.api.i_favorites(): + note = fav.get("note") + note["instance"] = self.instance + note["instance_remote"] = note["user"]["host"] + note["count"] = len(note["files"]) + note["date"] = text.parse_datetime( + note["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") + + yield Message.Directory, note + for note["num"], file in enumerate(note["files"], 1): + file["date"] = text.parse_datetime( + file["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") + note["file"] = file + url = file["url"] + yield Message.Url, url, text.nameext_from_url(url, note) + + class MisskeyAPI(): """Interface for Misskey API @@ -164,6 +191,7 @@ class MisskeyAPI(): self.root = extractor.root self.extractor = extractor self.headers = {"Content-Type": "application/json"} + self.access_token = extractor.config("access-token") def user_id_by_username(self, username): endpoint = "/users/show" @@ -187,6 +215,14 @@ class MisskeyAPI(): data = {"noteId": note_id} return self._call(endpoint, data) + def i_favorites(self): + endpoint = "/i/favorites" + data = {} + if not self.access_token: + raise exception.AuthenticationError() + data["i"] = self.access_token + return self._pagination(endpoint, data) + def _call(self, endpoint, data): url = self.root + "/api" + endpoint return self.extractor.request( From f9b7a033e02e9be5225f3c85fba31bfe08e7a059 Mon Sep 17 00:00:00 2001 From: Naatie Date: Wed, 26 Apr 2023 14:44:47 +0700 Subject: [PATCH 2/3] [misskey] refactor misskey extractor --- gallery_dl/extractor/misskey.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 7e5ef799..26b7f194 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -27,6 +27,8 @@ class MisskeyExtractor(BaseExtractor): def items(self): for note in self.notes(): + if "note" in note: + note = note["note"] files = note.pop("files") or [] renote = note.get("renote") if renote: @@ -157,26 +159,12 @@ class MisskeyMyFavoritesExtractor(MisskeyExtractor): subcategory = "favorites" pattern = BASE_PATTERN + r"(/my/favorites|/api/i/favorites)" test = ( - ("https://misskey.io/my/favorites",), - ("https://misskey.io/api/i/favorites",), + ("https://misskey.io/my/favorites"), + ("https://misskey.io/api/i/favorites"), ) - def items(self): - for fav in self.api.i_favorites(): - note = fav.get("note") - note["instance"] = self.instance - note["instance_remote"] = note["user"]["host"] - note["count"] = len(note["files"]) - note["date"] = text.parse_datetime( - note["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") - - yield Message.Directory, note - for note["num"], file in enumerate(note["files"], 1): - file["date"] = text.parse_datetime( - file["createdAt"], "%Y-%m-%dT%H:%M:%S.%f%z") - note["file"] = file - url = file["url"] - yield Message.Url, url, text.nameext_from_url(url, note) + def notes(self): + return self.api.i_favorites() class MisskeyAPI(): @@ -217,10 +205,9 @@ class MisskeyAPI(): def i_favorites(self): endpoint = "/i/favorites" - data = {} if not self.access_token: raise exception.AuthenticationError() - data["i"] = self.access_token + data = {"i": self.access_token} return self._pagination(endpoint, data) def _call(self, endpoint, data): From c76f0f3a1bbc63b726396e5a92c0f0638565b1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 23 May 2023 22:15:20 +0200 Subject: [PATCH 3/3] [misskey] update - rename to 'MisskeyFavoriteExtractor' - add 'access-token' option to docs - add test URLs for other instances - simplify 'pattern' --- docs/configuration.rst | 12 ++++++++++-- docs/gallery-dl.conf | 1 + docs/supportedsites.md | 6 +++--- gallery_dl/extractor/misskey.py | 12 +++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 6d5ea7c2..7a9c8082 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2059,8 +2059,16 @@ Description Also emit metadata for text-only posts without media content. +extractor.[misskey].access-token +-------------------------------- +Type + ``string`` +Description + Your access token, necessary to fetch favorited notes. + + extractor.[misskey].renotes ----------------------------- +--------------------------- Type ``bool`` Default @@ -2070,7 +2078,7 @@ Description extractor.[misskey].replies ----------------------------- +--------------------------- Type ``bool`` Default diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 92451fda..bc1985a2 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -190,6 +190,7 @@ "password": null }, "misskey": { + "access-token": null, "renotes": false, "replies": true }, diff --git a/docs/supportedsites.md b/docs/supportedsites.md index f98508ca..938e91da 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1132,19 +1132,19 @@ Consider all sites to be NSFW unless otherwise known. Misskey.io https://misskey.io/ - Images from Notes, User Profiles + Favorites, Images from Notes, User Profiles Lesbian.energy https://lesbian.energy/ - Images from Notes, User Profiles + Favorites, Images from Notes, User Profiles Sushi.ski https://sushi.ski/ - Images from Notes, User Profiles + Favorites, Images from Notes, User Profiles diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 26b7f194..37efac07 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -70,7 +70,7 @@ BASE_PATTERN = MisskeyExtractor.update({ }, "lesbian.energy": { "root": "https://lesbian.energy", - "pattern": r"lesbian\.energy" + "pattern": r"lesbian\.energy", }, "sushi.ski": { "root": "https://sushi.ski", @@ -154,13 +154,15 @@ class MisskeyNoteExtractor(MisskeyExtractor): return (self.api.notes_show(self.item),) -class MisskeyMyFavoritesExtractor(MisskeyExtractor): - """Extractor for images from favorites""" - subcategory = "favorites" - pattern = BASE_PATTERN + r"(/my/favorites|/api/i/favorites)" +class MisskeyFavoriteExtractor(MisskeyExtractor): + """Extractor for favorited notes""" + subcategory = "favorite" + pattern = BASE_PATTERN + r"/(?:my|api/i)/favorites" test = ( ("https://misskey.io/my/favorites"), ("https://misskey.io/api/i/favorites"), + ("https://lesbian.energy/my/favorites"), + ("https://sushi.ski/my/favorites"), ) def notes(self):