Merge pull request '[#37] fix frienderica pinned collection fetching' (#42) from friendica-featured-collection into develop
Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/42stable
commit
01652167d0
@ -0,0 +1,77 @@
|
||||
# Akkoma: The cooler fediverse server
|
||||
# Copyright © 2022- Akkoma Authors <https://akkoma.dev/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Akkoma.Collections.Fetcher do
|
||||
@moduledoc """
|
||||
Activitypub Collections fetching functions
|
||||
see: https://www.w3.org/TR/activitystreams-core/#paging
|
||||
"""
|
||||
alias Pleroma.Object.Fetcher
|
||||
alias Pleroma.Config
|
||||
|
||||
def fetch_collection_by_ap_id(ap_id) when is_binary(ap_id) do
|
||||
fetch_collection(ap_id)
|
||||
end
|
||||
|
||||
def fetch_collection(ap_id) when is_binary(ap_id) do
|
||||
with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
|
||||
{:ok, objects_from_collection(page)}
|
||||
else
|
||||
e ->
|
||||
Logger.error("Could not fetch collection #{ap_id} - #{inspect(e)}")
|
||||
e
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_collection(%{"type" => type} = page)
|
||||
when type in ["Collection", "OrderedCollection"] do
|
||||
{:ok, objects_from_collection(page)}
|
||||
end
|
||||
|
||||
defp items_in_page(%{"type" => type, "orderedItems" => items})
|
||||
when is_list(items) and type in ["OrderedCollection", "OrderedCollectionPage"],
|
||||
do: items
|
||||
|
||||
defp items_in_page(%{"type" => type, "items" => items})
|
||||
when is_list(items) and type in ["Collection", "CollectionPage"],
|
||||
do: items
|
||||
|
||||
defp objects_from_collection(%{"type" => "OrderedCollection", "orderedItems" => items})
|
||||
when is_list(items),
|
||||
do: items
|
||||
|
||||
defp objects_from_collection(%{"type" => "Collection", "items" => items}) when is_list(items),
|
||||
do: items
|
||||
|
||||
defp objects_from_collection(%{"type" => type, "first" => first})
|
||||
when is_binary(first) and type in ["Collection", "OrderedCollection"] do
|
||||
fetch_page_items(first)
|
||||
end
|
||||
|
||||
defp objects_from_collection(%{"type" => type, "first" => %{"id" => id}})
|
||||
when is_binary(id) and type in ["Collection", "OrderedCollection"] do
|
||||
fetch_page_items(id)
|
||||
end
|
||||
|
||||
defp fetch_page_items(id, items \\ []) do
|
||||
if Enum.count(items) >= Config.get([:activitypub, :max_collection_objects]) do
|
||||
items
|
||||
else
|
||||
{:ok, page} = Fetcher.fetch_and_contain_remote_object_from_id(id)
|
||||
objects = items_in_page(page)
|
||||
|
||||
if Enum.count(objects) > 0 do
|
||||
maybe_next_page(page, items ++ objects)
|
||||
else
|
||||
items
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_next_page(%{"next" => id}, items) when is_binary(id) do
|
||||
fetch_page_items(id, items)
|
||||
end
|
||||
|
||||
defp maybe_next_page(_, items), do: items
|
||||
end
|
@ -0,0 +1,19 @@
|
||||
{
|
||||
"@context": "https://www.w3.org/ns/activitystreams",
|
||||
"id": "https://example.com/collection/ordered_array",
|
||||
"summary": "Object history",
|
||||
"type": "OrderedCollection",
|
||||
"totalItems": 2,
|
||||
"orderedItems": [
|
||||
{
|
||||
"type": "Create",
|
||||
"actor": "http://www.test.example/sally",
|
||||
"object": "http://example.org/foo"
|
||||
},
|
||||
{
|
||||
"type": "Like",
|
||||
"actor": "http://www.test.example/joe",
|
||||
"object": "http://example.org/foo"
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,19 @@
|
||||
{
|
||||
"@context": "https://www.w3.org/ns/activitystreams",
|
||||
"id": "https://example.com/collection/unordered_array",
|
||||
"summary": "Object history",
|
||||
"type": "Collection",
|
||||
"totalItems": 2,
|
||||
"items": [
|
||||
{
|
||||
"type": "Create",
|
||||
"actor": "http://www.test.example/sally",
|
||||
"object": "http://example.org/foo"
|
||||
},
|
||||
{
|
||||
"type": "Like",
|
||||
"actor": "http://www.test.example/joe",
|
||||
"object": "http://example.org/foo"
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,20 @@
|
||||
{
|
||||
"@context": "https://www.w3.org/ns/activitystreams",
|
||||
"summary": "Sally's recent activities",
|
||||
"type": "Collection",
|
||||
"id": "http://example.org/foo",
|
||||
"totalItems": 10,
|
||||
"first": {
|
||||
"type": "CollectionPage",
|
||||
"id": "http://example.org/foo?page=1",
|
||||
"partOf": "http://example.org/foo",
|
||||
"next": "http://example.org/foo?page=2",
|
||||
"items": [
|
||||
{
|
||||
"type": "Create",
|
||||
"actor": "http://www.test.example/sally",
|
||||
"object": "http://example.org/foo"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
@ -0,0 +1,13 @@
|
||||
{
|
||||
"type": "CollectionPage",
|
||||
"id": "https://example.com/collection/unordered_page_reference?page=1",
|
||||
"partOf": "https://example.com/collection/unordered_page_reference",
|
||||
"next": "https://example.com/collection/unordered_page_reference?page=2",
|
||||
"items": [
|
||||
{
|
||||
"type": "Create",
|
||||
"actor": "http://www.test.example/sally",
|
||||
"object": "http://example.org/foo"
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,8 @@
|
||||
{
|
||||
"@context": "https://www.w3.org/ns/activitystreams",
|
||||
"summary": "Sally's recent activities",
|
||||
"type": "Collection",
|
||||
"id": "https://example.com/collection/unordered_page_reference",
|
||||
"totalItems": 10,
|
||||
"first": "https://example.com/collection/unordered_page_reference?page=1"
|
||||
}
|
@ -0,0 +1,12 @@
|
||||
{
|
||||
"type": "CollectionPage",
|
||||
"id": "https://example.com/collection/unordered_page_reference?page=2",
|
||||
"partOf": "https://example.com/collection/unordered_page_reference",
|
||||
"items": [
|
||||
{
|
||||
"type": "Like",
|
||||
"actor": "http://www.test.example/sally",
|
||||
"object": "http://example.org/foo"
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,29 @@
|
||||
{
|
||||
"@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
{
|
||||
"vcard": "http://www.w3.org/2006/vcard/ns#",
|
||||
"dfrn": "http://purl.org/macgirvin/dfrn/1.0/",
|
||||
"diaspora": "https://diasporafoundation.org/ns/",
|
||||
"litepub": "http://litepub.social/ns#",
|
||||
"toot": "http://joinmastodon.org/ns#",
|
||||
"featured": {
|
||||
"@id": "toot:featured",
|
||||
"@type": "@id"
|
||||
},
|
||||
"schema": "http://schema.org#",
|
||||
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
|
||||
"sensitive": "as:sensitive",
|
||||
"Hashtag": "as:Hashtag",
|
||||
"directMessage": "litepub:directMessage",
|
||||
"discoverable": "toot:discoverable",
|
||||
"PropertyValue": "schema:PropertyValue",
|
||||
"value": "schema:value"
|
||||
}
|
||||
],
|
||||
"id": "https://friendica.example.com/featured/raha",
|
||||
"type": "OrderedCollection",
|
||||
"totalItems": 0,
|
||||
"first": "https://friendica.example.com/featured/raha?page=1"
|
||||
}
|
@ -0,0 +1,34 @@
|
||||
{
|
||||
"@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
{
|
||||
"vcard": "http://www.w3.org/2006/vcard/ns#",
|
||||
"dfrn": "http://purl.org/macgirvin/dfrn/1.0/",
|
||||
"diaspora": "https://diasporafoundation.org/ns/",
|
||||
"litepub": "http://litepub.social/ns#",
|
||||
"toot": "http://joinmastodon.org/ns#",
|
||||
"featured": {
|
||||
"@id": "toot:featured",
|
||||
"@type": "@id"
|
||||
},
|
||||
"schema": "http://schema.org#",
|
||||
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
|
||||
"sensitive": "as:sensitive",
|
||||
"Hashtag": "as:Hashtag",
|
||||
"directMessage": "litepub:directMessage",
|
||||
"discoverable": "toot:discoverable",
|
||||
"PropertyValue": "schema:PropertyValue",
|
||||
"value": "schema:value"
|
||||
}
|
||||
],
|
||||
"id": "https://friendica.example.com/featured/raha?page=1",
|
||||
"type": "OrderedCollectionPage",
|
||||
"totalItems": 0,
|
||||
"partOf": "https://friendica.example.com/featured/raha",
|
||||
"orderedItems": [
|
||||
{
|
||||
"id": "http://inserted"
|
||||
}
|
||||
]
|
||||
}
|
@ -0,0 +1,167 @@
|
||||
defmodule Akkoma.Collections.FetcherTest do
|
||||
use Pleroma.DataCase
|
||||
use Oban.Testing, repo: Pleroma.Repo
|
||||
|
||||
alias Akkoma.Collections.Fetcher
|
||||
|
||||
import Tesla.Mock
|
||||
|
||||
setup do
|
||||
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
|
||||
:ok
|
||||
end
|
||||
|
||||
test "it should extract items from an embedded array in a Collection" do
|
||||
unordered_collection =
|
||||
"test/fixtures/collections/unordered_array.json"
|
||||
|> File.read!()
|
||||
|
||||
ap_id = "https://example.com/collection/ordered_array"
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{
|
||||
method: :get,
|
||||
url: ^ap_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: unordered_collection,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
end)
|
||||
|
||||
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
|
||||
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
|
||||
end
|
||||
|
||||
test "it should extract items from an embedded array in an OrderedCollection" do
|
||||
ordered_collection =
|
||||
"test/fixtures/collections/ordered_array.json"
|
||||
|> File.read!()
|
||||
|
||||
ap_id = "https://example.com/collection/ordered_array"
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{
|
||||
method: :get,
|
||||
url: ^ap_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: ordered_collection,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
end)
|
||||
|
||||
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
|
||||
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
|
||||
end
|
||||
|
||||
test "it should extract items from an referenced first page in a Collection" do
|
||||
unordered_collection =
|
||||
"test/fixtures/collections/unordered_page_reference.json"
|
||||
|> File.read!()
|
||||
|
||||
first_page =
|
||||
"test/fixtures/collections/unordered_page_first.json"
|
||||
|> File.read!()
|
||||
|
||||
second_page =
|
||||
"test/fixtures/collections/unordered_page_second.json"
|
||||
|> File.read!()
|
||||
|
||||
ap_id = "https://example.com/collection/unordered_page_reference"
|
||||
first_page_id = "https://example.com/collection/unordered_page_reference?page=1"
|
||||
second_page_id = "https://example.com/collection/unordered_page_reference?page=2"
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{
|
||||
method: :get,
|
||||
url: ^ap_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: unordered_collection,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
|
||||
%{
|
||||
method: :get,
|
||||
url: ^first_page_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: first_page,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
|
||||
%{
|
||||
method: :get,
|
||||
url: ^second_page_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: second_page,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
end)
|
||||
|
||||
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
|
||||
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
|
||||
end
|
||||
|
||||
test "it should stop fetching when we hit :max_collection_objects" do
|
||||
clear_config([:activitypub, :max_collection_objects], 1)
|
||||
|
||||
unordered_collection =
|
||||
"test/fixtures/collections/unordered_page_reference.json"
|
||||
|> File.read!()
|
||||
|
||||
first_page =
|
||||
"test/fixtures/collections/unordered_page_first.json"
|
||||
|> File.read!()
|
||||
|
||||
second_page =
|
||||
"test/fixtures/collections/unordered_page_second.json"
|
||||
|> File.read!()
|
||||
|
||||
ap_id = "https://example.com/collection/unordered_page_reference"
|
||||
first_page_id = "https://example.com/collection/unordered_page_reference?page=1"
|
||||
second_page_id = "https://example.com/collection/unordered_page_reference?page=2"
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{
|
||||
method: :get,
|
||||
url: ^ap_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: unordered_collection,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
|
||||
%{
|
||||
method: :get,
|
||||
url: ^first_page_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: first_page,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
|
||||
%{
|
||||
method: :get,
|
||||
url: ^second_page_id
|
||||
} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: second_page,
|
||||
headers: [{"content-type", "application/activity+json"}]
|
||||
}
|
||||
end)
|
||||
|
||||
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
|
||||
assert [%{"type" => "Create"}] = objects
|
||||
end
|
||||
end
|
Loading…
Reference in new issue