Pipeline Ingestion: Video and Article See merge request pleroma/pleroma!2908stable
commit
adbd0032fa
@ -0,0 +1,134 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
|
||||
use Ecto.Schema
|
||||
|
||||
alias Pleroma.EarmarkRenderer
|
||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
|
||||
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
|
||||
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
|
||||
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key false
|
||||
@derive Jason.Encoder
|
||||
|
||||
embedded_schema do
|
||||
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
||||
field(:to, ObjectValidators.Recipients, default: [])
|
||||
field(:cc, ObjectValidators.Recipients, default: [])
|
||||
field(:bto, ObjectValidators.Recipients, default: [])
|
||||
field(:bcc, ObjectValidators.Recipients, default: [])
|
||||
# TODO: Write type
|
||||
field(:tag, {:array, :map}, default: [])
|
||||
field(:type, :string)
|
||||
|
||||
field(:name, :string)
|
||||
field(:summary, :string)
|
||||
field(:content, :string)
|
||||
|
||||
field(:context, :string)
|
||||
# short identifier for PleromaFE to group statuses by context
|
||||
field(:context_id, :integer)
|
||||
|
||||
# TODO: Remove actor on objects
|
||||
field(:actor, ObjectValidators.ObjectID)
|
||||
|
||||
field(:attributedTo, ObjectValidators.ObjectID)
|
||||
field(:published, ObjectValidators.DateTime)
|
||||
field(:emoji, ObjectValidators.Emoji, default: %{})
|
||||
field(:sensitive, :boolean, default: false)
|
||||
embeds_many(:attachment, AttachmentValidator)
|
||||
field(:replies_count, :integer, default: 0)
|
||||
field(:like_count, :integer, default: 0)
|
||||
field(:announcement_count, :integer, default: 0)
|
||||
field(:inReplyTo, ObjectValidators.ObjectID)
|
||||
field(:url, ObjectValidators.Uri)
|
||||
|
||||
field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
|
||||
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
|
||||
end
|
||||
|
||||
def cast_and_apply(data) do
|
||||
data
|
||||
|> cast_data
|
||||
|> apply_action(:insert)
|
||||
end
|
||||
|
||||
def cast_and_validate(data) do
|
||||
data
|
||||
|> cast_data()
|
||||
|> validate_data()
|
||||
end
|
||||
|
||||
def cast_data(data) do
|
||||
%__MODULE__{}
|
||||
|> changeset(data)
|
||||
end
|
||||
|
||||
defp fix_url(%{"url" => url} = data) when is_list(url) do
|
||||
attachment =
|
||||
Enum.find(url, fn x ->
|
||||
mime_type = x["mimeType"] || x["mediaType"] || ""
|
||||
|
||||
is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"])
|
||||
end)
|
||||
|
||||
link_element =
|
||||
Enum.find(url, fn x ->
|
||||
mime_type = x["mimeType"] || x["mediaType"] || ""
|
||||
|
||||
is_map(x) and mime_type == "text/html"
|
||||
end)
|
||||
|
||||
data
|
||||
|> Map.put("attachment", [attachment])
|
||||
|> Map.put("url", link_element["href"])
|
||||
end
|
||||
|
||||
defp fix_url(data), do: data
|
||||
|
||||
defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
|
||||
when is_binary(content) do
|
||||
content =
|
||||
content
|
||||
|> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
|
||||
|> Pleroma.HTML.filter_tags()
|
||||
|
||||
Map.put(data, "content", content)
|
||||
end
|
||||
|
||||
defp fix_content(data), do: data
|
||||
|
||||
defp fix(data) do
|
||||
data
|
||||
|> CommonFixes.fix_defaults()
|
||||
|> CommonFixes.fix_attribution()
|
||||
|> CommonFixes.fix_actor()
|
||||
|> Transmogrifier.fix_emoji()
|
||||
|> fix_url()
|
||||
|> fix_content()
|
||||
end
|
||||
|
||||
def changeset(struct, data) do
|
||||
data = fix(data)
|
||||
|
||||
struct
|
||||
|> cast(data, __schema__(:fields) -- [:attachment])
|
||||
|> cast_embed(:attachment)
|
||||
end
|
||||
|
||||
def validate_data(data_cng) do
|
||||
data_cng
|
||||
|> validate_inclusion(:type, ["Audio", "Video"])
|
||||
|> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
|
||||
|> CommonValidations.validate_any_presence([:cc, :to])
|
||||
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|
||||
|> CommonValidations.validate_actor_presence()
|
||||
|> CommonValidations.validate_host_match()
|
||||
end
|
||||
end
|
@ -1,73 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
|
||||
use Ecto.Schema
|
||||
|
||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key false
|
||||
|
||||
embedded_schema do
|
||||
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
||||
field(:to, ObjectValidators.Recipients, default: [])
|
||||
field(:cc, ObjectValidators.Recipients, default: [])
|
||||
field(:bto, ObjectValidators.Recipients, default: [])
|
||||
field(:bcc, ObjectValidators.Recipients, default: [])
|
||||
# TODO: Write type
|
||||
field(:tag, {:array, :map}, default: [])
|
||||
field(:type, :string)
|
||||
|
||||
field(:name, :string)
|
||||
field(:summary, :string)
|
||||
field(:content, :string)
|
||||
|
||||
field(:context, :string)
|
||||
# short identifier for PleromaFE to group statuses by context
|
||||
field(:context_id, :integer)
|
||||
|
||||
field(:actor, ObjectValidators.ObjectID)
|
||||
field(:attributedTo, ObjectValidators.ObjectID)
|
||||
field(:published, ObjectValidators.DateTime)
|
||||
field(:emoji, ObjectValidators.Emoji, default: %{})
|
||||
field(:sensitive, :boolean, default: false)
|
||||
# TODO: Write type
|
||||
field(:attachment, {:array, :map}, default: [])
|
||||
field(:replies_count, :integer, default: 0)
|
||||
field(:like_count, :integer, default: 0)
|
||||
field(:announcement_count, :integer, default: 0)
|
||||
field(:inReplyTo, ObjectValidators.ObjectID)
|
||||
field(:url, ObjectValidators.Uri)
|
||||
|
||||
field(:likes, {:array, :string}, default: [])
|
||||
field(:announcements, {:array, :string}, default: [])
|
||||
end
|
||||
|
||||
def cast_and_validate(data) do
|
||||
data
|
||||
|> cast_data()
|
||||
|> validate_data()
|
||||
end
|
||||
|
||||
defp fix(data) do
|
||||
data
|
||||
|> Transmogrifier.fix_emoji()
|
||||
end
|
||||
|
||||
def cast_data(data) do
|
||||
data = fix(data)
|
||||
|
||||
%__MODULE__{}
|
||||
|> cast(data, __schema__(:fields))
|
||||
end
|
||||
|
||||
def validate_data(data_cng) do
|
||||
data_cng
|
||||
|> validate_inclusion(:type, ["Note"])
|
||||
|> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
|
||||
end
|
||||
end
|
@ -1,24 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
|
||||
use Ecto.Schema
|
||||
|
||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||
|
||||
import Ecto.Changeset
|
||||
@primary_key false
|
||||
|
||||
embedded_schema do
|
||||
field(:type, :string)
|
||||
field(:href, ObjectValidators.Uri)
|
||||
field(:mediaType, :string, default: "application/octet-stream")
|
||||
end
|
||||
|
||||
def changeset(struct, data) do
|
||||
struct
|
||||
|> cast(data, __schema__(:fields))
|
||||
|> validate_required([:type, :href, :mediaType])
|
||||
end
|
||||
end
|
File diff suppressed because one or more lines are too long
@ -0,0 +1 @@
|
||||
{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Create","actor":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","object":{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Article","name":"The end is near: Mastodon plans to drop OStatus support","content":"<!-- wp:paragraph {\"dropCap\":true} -->\n<p class=\"has-drop-cap\">The days of OStatus are numbered. The venerable protocol has served as a glue between many different types of servers since the early days of the Fediverse, connecting StatusNet (now GNU Social) to Friendica, Hubzilla, Mastodon, and Pleroma.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Now that many fediverse platforms support ActivityPub as a successor protocol, Mastodon appears to be drawing a line in the sand. In <a href=\"https:\/\/www.patreon.com\/posts\/mastodon-2-9-and-28121681\">a Patreon update<\/a>, Eugen Rochko writes:<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:quote -->\n<blockquote class=\"wp-block-quote\"><p>...OStatus...has overstayed its welcome in the code...and now that most of the network uses ActivityPub, it's time for it to go. <\/p><cite>Eugen Rochko, Mastodon creator<\/cite><\/blockquote>\n<!-- \/wp:quote -->\n\n<!-- wp:paragraph -->\n<p>The <a href=\"https:\/\/github.com\/tootsuite\/mastodon\/pull\/11205\">pull request<\/a> to remove Pubsubhubbub and Salmon, two of the main components of OStatus, has already been merged into Mastodon's master branch.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Some projects will be left in the dark as a side effect of this. GNU Social and PostActiv, for example, both only communicate using OStatus. While <a href=\"https:\/\/mastodon.social\/@dansup\/102076573310057902\">some discussion<\/a> exists regarding adopting ActivityPub for GNU Social, and <a href=\"https:\/\/notabug.org\/diogo\/gnu-social\/src\/activitypub\/plugins\/ActivityPub\">a plugin is in development<\/a>, it hasn't been formally adopted yet. We just hope that the <a href=\"https:\/\/status.fsf.org\/main\/public\">Free Software Foundation's instance<\/a> gets updated in time!<\/p>\n<!-- \/wp:paragraph -->","summary":"One of the largest platforms in the federated social web is dropping the protocol that it started with.","attributedTo":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","url":"https:\/\/wedistribute.org\/2019\/07\/mastodon-drops-ostatus\/","to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810","likes":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/likes","shares":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/shares"},"to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85809"}
|
@ -0,0 +1,75 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do
|
||||
use Oban.Testing, repo: Pleroma.Repo
|
||||
use Pleroma.DataCase
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.Object.Fetcher
|
||||
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||
|
||||
test "Pterotype (Wordpress Plugin) Article" do
|
||||
Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} ->
|
||||
%Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")}
|
||||
end)
|
||||
|
||||
data =
|
||||
File.read!("test/fixtures/tesla_mock/wedistribute-create-article.json") |> Jason.decode!()
|
||||
|
||||
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
|
||||
|
||||
object = Object.normalize(data["object"])
|
||||
|
||||
assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support"
|
||||
|
||||
assert object.data["summary"] ==
|
||||
"One of the largest platforms in the federated social web is dropping the protocol that it started with."
|
||||
|
||||
assert object.data["url"] == "https://wedistribute.org/2019/07/mastodon-drops-ostatus/"
|
||||
end
|
||||
|
||||
test "Plume Article" do
|
||||
Tesla.Mock.mock(fn
|
||||
%{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
|
||||
}
|
||||
|
||||
%{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
|
||||
}
|
||||
end)
|
||||
|
||||
{:ok, object} =
|
||||
Fetcher.fetch_object_from_id(
|
||||
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
|
||||
)
|
||||
|
||||
assert object.data["name"] == "This Month in Plume: June 2018"
|
||||
|
||||
assert object.data["url"] ==
|
||||
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
|
||||
end
|
||||
|
||||
test "Prismo Article" do
|
||||
Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
|
||||
}
|
||||
end)
|
||||
|
||||
data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!()
|
||||
|
||||
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
|
||||
object = Object.normalize(data["object"])
|
||||
|
||||
assert object.data["url"] == "https://prismo.news/posts/83"
|
||||
end
|
||||
end
|
Loading…
Reference in new issue