commit
3be08e7c2e
@ -0,0 +1,45 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Activity.HTML do
|
||||||
|
alias Pleroma.HTML
|
||||||
|
alias Pleroma.Object
|
||||||
|
|
||||||
|
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
|
||||||
|
|
||||||
|
def get_cached_scrubbed_html_for_activity(
|
||||||
|
content,
|
||||||
|
scrubbers,
|
||||||
|
activity,
|
||||||
|
key \\ "",
|
||||||
|
callback \\ fn x -> x end
|
||||||
|
) do
|
||||||
|
key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
|
||||||
|
|
||||||
|
@cachex.fetch!(:scrubber_cache, key, fn _key ->
|
||||||
|
object = Object.normalize(activity, fetch: false)
|
||||||
|
HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_cached_stripped_html_for_activity(content, activity, key) do
|
||||||
|
get_cached_scrubbed_html_for_activity(
|
||||||
|
content,
|
||||||
|
FastSanitize.Sanitizer.StripTags,
|
||||||
|
activity,
|
||||||
|
key,
|
||||||
|
&HtmlEntities.decode/1
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
|
||||||
|
generate_scrubber_signature([scrubber])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp generate_scrubber_signature(scrubbers) do
|
||||||
|
Enum.reduce(scrubbers, "", fn scrubber, signature ->
|
||||||
|
"#{signature}#{to_string(scrubber)}"
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,45 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.DataMigration do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
alias Pleroma.DataMigration
|
||||||
|
alias Pleroma.DataMigration.State
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
import Ecto.Changeset
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
schema "data_migrations" do
|
||||||
|
field(:name, :string)
|
||||||
|
field(:state, State, default: :pending)
|
||||||
|
field(:feature_lock, :boolean, default: false)
|
||||||
|
field(:params, :map, default: %{})
|
||||||
|
field(:data, :map, default: %{})
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
def changeset(data_migration, params \\ %{}) do
|
||||||
|
data_migration
|
||||||
|
|> cast(params, [:name, :state, :feature_lock, :params, :data])
|
||||||
|
|> validate_required([:name])
|
||||||
|
|> unique_constraint(:name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_one_by_id(id, params \\ %{}) do
|
||||||
|
with {1, _} <-
|
||||||
|
from(dm in DataMigration, where: dm.id == ^id)
|
||||||
|
|> Repo.update_all(set: params) do
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_by_name(name) do
|
||||||
|
Repo.get_by(DataMigration, name: name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
|
||||||
|
end
|
@ -1,256 +0,0 @@
|
|||||||
# Pleroma: A lightweight social networking server
|
|
||||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
#
|
|
||||||
# This file is derived from Earmark, under the following copyright:
|
|
||||||
# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
|
|
||||||
defmodule Pleroma.EarmarkRenderer do
|
|
||||||
@moduledoc false
|
|
||||||
|
|
||||||
alias Earmark.Block
|
|
||||||
alias Earmark.Context
|
|
||||||
alias Earmark.HtmlRenderer
|
|
||||||
alias Earmark.Options
|
|
||||||
|
|
||||||
import Earmark.Inline, only: [convert: 3]
|
|
||||||
import Earmark.Helpers.HtmlHelpers
|
|
||||||
import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
|
|
||||||
import Earmark.Context, only: [append: 2, set_value: 2]
|
|
||||||
import Earmark.Options, only: [get_mapper: 1]
|
|
||||||
|
|
||||||
@doc false
|
|
||||||
def render(blocks, %Context{options: %Options{}} = context) do
|
|
||||||
messages = get_messages(context)
|
|
||||||
|
|
||||||
{contexts, html} =
|
|
||||||
get_mapper(context.options).(
|
|
||||||
blocks,
|
|
||||||
&render_block(&1, put_in(context.options.messages, []))
|
|
||||||
)
|
|
||||||
|> Enum.unzip()
|
|
||||||
|
|
||||||
all_messages =
|
|
||||||
contexts
|
|
||||||
|> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
|
|
||||||
|
|
||||||
{put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
|
|
||||||
end
|
|
||||||
|
|
||||||
#############
|
|
||||||
# Paragraph #
|
|
||||||
#############
|
|
||||||
defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
|
|
||||||
lines = convert(lines, lnb, context)
|
|
||||||
add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
########
|
|
||||||
# Html #
|
|
||||||
########
|
|
||||||
defp render_block(%Block.Html{html: html}, context) do
|
|
||||||
{context, html}
|
|
||||||
end
|
|
||||||
|
|
||||||
defp render_block(%Block.HtmlComment{lines: lines}, context) do
|
|
||||||
{context, lines}
|
|
||||||
end
|
|
||||||
|
|
||||||
defp render_block(%Block.HtmlOneline{html: html}, context) do
|
|
||||||
{context, html}
|
|
||||||
end
|
|
||||||
|
|
||||||
#########
|
|
||||||
# Ruler #
|
|
||||||
#########
|
|
||||||
defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
|
|
||||||
add_attrs(context, "<hr />", attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
###########
|
|
||||||
# Heading #
|
|
||||||
###########
|
|
||||||
defp render_block(
|
|
||||||
%Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
|
|
||||||
context
|
|
||||||
) do
|
|
||||||
converted = convert(content, lnb, context)
|
|
||||||
html = "<h#{level}>#{converted.value}</h#{level}>"
|
|
||||||
add_attrs(converted, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
##############
|
|
||||||
# Blockquote #
|
|
||||||
##############
|
|
||||||
|
|
||||||
defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
|
|
||||||
{context1, body} = render(blocks, context)
|
|
||||||
html = "<blockquote>#{body}</blockquote>"
|
|
||||||
add_attrs(context1, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
#########
|
|
||||||
# Table #
|
|
||||||
#########
|
|
||||||
|
|
||||||
defp render_block(
|
|
||||||
%Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
|
|
||||||
context
|
|
||||||
) do
|
|
||||||
{context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
|
|
||||||
context2 = set_value(context1, html)
|
|
||||||
|
|
||||||
context3 =
|
|
||||||
if header do
|
|
||||||
append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
|
|
||||||
else
|
|
||||||
# Maybe an error, needed append(context, html)
|
|
||||||
context2
|
|
||||||
end
|
|
||||||
|
|
||||||
context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
|
|
||||||
|
|
||||||
{context4, [context4.value, "</table>"]}
|
|
||||||
end
|
|
||||||
|
|
||||||
########
|
|
||||||
# Code #
|
|
||||||
########
|
|
||||||
|
|
||||||
defp render_block(
|
|
||||||
%Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
|
|
||||||
%Context{options: options} = context
|
|
||||||
) do
|
|
||||||
class =
|
|
||||||
if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
|
|
||||||
|
|
||||||
tag = ~s[<pre><code#{class}>]
|
|
||||||
lines = options.render_code.(block)
|
|
||||||
html = ~s[#{tag}#{lines}</code></pre>]
|
|
||||||
add_attrs(context, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
#########
|
|
||||||
# Lists #
|
|
||||||
#########
|
|
||||||
|
|
||||||
defp render_block(
|
|
||||||
%Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
|
|
||||||
context
|
|
||||||
) do
|
|
||||||
{context1, content} = render(items, context)
|
|
||||||
html = "<#{type}#{start}>#{content}</#{type}>"
|
|
||||||
add_attrs(context1, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
# format a single paragraph list item, and remove the para tags
|
|
||||||
defp render_block(
|
|
||||||
%Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
|
|
||||||
context
|
|
||||||
)
|
|
||||||
when length(blocks) == 1 do
|
|
||||||
{context1, content} = render(blocks, context)
|
|
||||||
content = Regex.replace(~r{</?p>}, content, "")
|
|
||||||
html = "<li>#{content}</li>"
|
|
||||||
add_attrs(context1, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
# format a spaced list item
|
|
||||||
defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
|
|
||||||
{context1, content} = render(blocks, context)
|
|
||||||
html = "<li>#{content}</li>"
|
|
||||||
add_attrs(context1, html, attrs, [], lnb)
|
|
||||||
end
|
|
||||||
|
|
||||||
##################
|
|
||||||
# Footnote Block #
|
|
||||||
##################
|
|
||||||
|
|
||||||
defp render_block(%Block.FnList{blocks: footnotes}, context) do
|
|
||||||
items =
|
|
||||||
Enum.map(footnotes, fn note ->
|
|
||||||
blocks = append_footnote_link(note)
|
|
||||||
%Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
|
|
||||||
end)
|
|
||||||
|
|
||||||
{context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
|
|
||||||
{context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
|
|
||||||
end
|
|
||||||
|
|
||||||
#######################################
|
|
||||||
# Isolated IALs are rendered as paras #
|
|
||||||
#######################################
|
|
||||||
|
|
||||||
defp render_block(%Block.Ial{verbatim: verbatim}, context) do
|
|
||||||
{context, "<p>{:#{verbatim}}</p>"}
|
|
||||||
end
|
|
||||||
|
|
||||||
####################
|
|
||||||
# IDDef is ignored #
|
|
||||||
####################
|
|
||||||
|
|
||||||
defp render_block(%Block.IdDef{}, context), do: {context, ""}
|
|
||||||
|
|
||||||
#####################################
|
|
||||||
# And here are the inline renderers #
|
|
||||||
#####################################
|
|
||||||
|
|
||||||
defdelegate br, to: HtmlRenderer
|
|
||||||
defdelegate codespan(text), to: HtmlRenderer
|
|
||||||
defdelegate em(text), to: HtmlRenderer
|
|
||||||
defdelegate strong(text), to: HtmlRenderer
|
|
||||||
defdelegate strikethrough(text), to: HtmlRenderer
|
|
||||||
|
|
||||||
defdelegate link(url, text), to: HtmlRenderer
|
|
||||||
defdelegate link(url, text, title), to: HtmlRenderer
|
|
||||||
|
|
||||||
defdelegate image(path, alt, title), to: HtmlRenderer
|
|
||||||
|
|
||||||
defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
|
|
||||||
|
|
||||||
# Table rows
|
|
||||||
defp add_trs(context, rows, tag, aligns, lnb) do
|
|
||||||
numbered_rows =
|
|
||||||
rows
|
|
||||||
|> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
|
|
||||||
|
|
||||||
numbered_rows
|
|
||||||
|> Enum.reduce(context, fn {row, lnb}, ctx ->
|
|
||||||
append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
|
|
||||||
end)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp add_tds(context, row, tag, aligns, lnb) do
|
|
||||||
Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
|
|
||||||
end
|
|
||||||
|
|
||||||
defp add_td_fn(row, tag, aligns, lnb) do
|
|
||||||
fn n, ctx ->
|
|
||||||
style =
|
|
||||||
case Enum.at(aligns, n - 1, :default) do
|
|
||||||
:default -> ""
|
|
||||||
align -> " style=\"text-align: #{align}\""
|
|
||||||
end
|
|
||||||
|
|
||||||
col = Enum.at(row, n - 1)
|
|
||||||
converted = convert(col, lnb, set_messages(ctx, []))
|
|
||||||
append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
###############################
|
|
||||||
# Append Footnote Return Link #
|
|
||||||
###############################
|
|
||||||
|
|
||||||
defdelegate append_footnote_link(note), to: HtmlRenderer
|
|
||||||
defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
|
|
||||||
|
|
||||||
defdelegate render_code(lines), to: HtmlRenderer
|
|
||||||
|
|
||||||
defp code_classes(language, prefix) do
|
|
||||||
["" | String.split(prefix || "")]
|
|
||||||
|> Enum.map(fn pfx -> "#{pfx}#{language}" end)
|
|
||||||
|> Enum.join(" ")
|
|
||||||
end
|
|
||||||
end
|
|
@ -0,0 +1,106 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Hashtag do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
import Ecto.Changeset
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias Ecto.Multi
|
||||||
|
alias Pleroma.Hashtag
|
||||||
|
alias Pleroma.Object
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
schema "hashtags" do
|
||||||
|
field(:name, :string)
|
||||||
|
|
||||||
|
many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
def normalize_name(name) do
|
||||||
|
name
|
||||||
|
|> String.downcase()
|
||||||
|
|> String.trim()
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_or_create_by_name(name) do
|
||||||
|
changeset = changeset(%Hashtag{}, %{name: name})
|
||||||
|
|
||||||
|
Repo.insert(
|
||||||
|
changeset,
|
||||||
|
on_conflict: [set: [name: get_field(changeset, :name)]],
|
||||||
|
conflict_target: :name,
|
||||||
|
returning: true
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_or_create_by_names(names) when is_list(names) do
|
||||||
|
names = Enum.map(names, &normalize_name/1)
|
||||||
|
timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
|
||||||
|
|
||||||
|
structs =
|
||||||
|
Enum.map(names, fn name ->
|
||||||
|
%Hashtag{}
|
||||||
|
|> changeset(%{name: name})
|
||||||
|
|> Map.get(:changes)
|
||||||
|
|> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
|
||||||
|
end)
|
||||||
|
|
||||||
|
try do
|
||||||
|
with {:ok, %{query_op: hashtags}} <-
|
||||||
|
Multi.new()
|
||||||
|
|> Multi.insert_all(:insert_all_op, Hashtag, structs,
|
||||||
|
on_conflict: :nothing,
|
||||||
|
conflict_target: :name
|
||||||
|
)
|
||||||
|
|> Multi.run(:query_op, fn _repo, _changes ->
|
||||||
|
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
|
||||||
|
end)
|
||||||
|
|> Repo.transaction() do
|
||||||
|
{:ok, hashtags}
|
||||||
|
else
|
||||||
|
{:error, _name, value, _changes_so_far} -> {:error, value}
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
e -> {:error, e}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def changeset(%Hashtag{} = struct, params) do
|
||||||
|
struct
|
||||||
|
|> cast(params, [:name])
|
||||||
|
|> update_change(:name, &normalize_name/1)
|
||||||
|
|> validate_required([:name])
|
||||||
|
|> unique_constraint(:name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def unlink(%Object{id: object_id}) do
|
||||||
|
with {_, hashtag_ids} <-
|
||||||
|
from(hto in "hashtags_objects",
|
||||||
|
where: hto.object_id == ^object_id,
|
||||||
|
select: hto.hashtag_id
|
||||||
|
)
|
||||||
|
|> Repo.delete_all(),
|
||||||
|
{:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
|
||||||
|
{:ok, length(hashtag_ids), unreferenced_count}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@delete_unreferenced_query """
|
||||||
|
DELETE FROM hashtags WHERE id IN
|
||||||
|
(SELECT hashtags.id FROM hashtags
|
||||||
|
LEFT OUTER JOIN hashtags_objects
|
||||||
|
ON hashtags_objects.hashtag_id = hashtags.id
|
||||||
|
WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
|
||||||
|
"""
|
||||||
|
|
||||||
|
def delete_unreferenced(ids) do
|
||||||
|
with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
|
||||||
|
{:ok, deleted_count}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,208 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.HashtagsTableMigrator do
|
||||||
|
defmodule State do
|
||||||
|
use Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
|
||||||
|
@impl Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table
|
||||||
|
end
|
||||||
|
|
||||||
|
use Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
|
||||||
|
alias Pleroma.Hashtag
|
||||||
|
alias Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
alias Pleroma.Object
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def feature_config_path, do: [:features, :improved_hashtag_timeline]
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0)
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def perform do
|
||||||
|
data_migration_id = data_migration_id()
|
||||||
|
max_processed_id = get_stat(:max_processed_id, 0)
|
||||||
|
|
||||||
|
Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...")
|
||||||
|
|
||||||
|
query()
|
||||||
|
|> where([object], object.id > ^max_processed_id)
|
||||||
|
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|
||||||
|
|> Stream.each(fn objects ->
|
||||||
|
object_ids = Enum.map(objects, & &1.id)
|
||||||
|
|
||||||
|
results = Enum.map(objects, &transfer_object_hashtags(&1))
|
||||||
|
|
||||||
|
failed_ids =
|
||||||
|
results
|
||||||
|
|> Enum.filter(&(elem(&1, 0) == :error))
|
||||||
|
|> Enum.map(&elem(&1, 1))
|
||||||
|
|
||||||
|
# Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags
|
||||||
|
chunk_affected_count =
|
||||||
|
results
|
||||||
|
|> Enum.filter(&(elem(&1, 0) == :ok))
|
||||||
|
|> length()
|
||||||
|
|
||||||
|
for failed_id <- failed_ids do
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
|
||||||
|
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
|
||||||
|
[data_migration_id, failed_id]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"DELETE FROM data_migration_failed_ids " <>
|
||||||
|
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
|
||||||
|
[data_migration_id, object_ids -- failed_ids]
|
||||||
|
)
|
||||||
|
|
||||||
|
max_object_id = Enum.at(object_ids, -1)
|
||||||
|
|
||||||
|
put_stat(:max_processed_id, max_object_id)
|
||||||
|
increment_stat(:iteration_processed_count, length(object_ids))
|
||||||
|
increment_stat(:processed_count, length(object_ids))
|
||||||
|
increment_stat(:failed_count, length(failed_ids))
|
||||||
|
increment_stat(:affected_count, chunk_affected_count)
|
||||||
|
put_stat(:records_per_second, records_per_second())
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
# A quick and dirty approach to controlling the load this background migration imposes
|
||||||
|
sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
|
||||||
|
Process.sleep(sleep_interval)
|
||||||
|
end)
|
||||||
|
|> Stream.run()
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def query do
|
||||||
|
# Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
|
||||||
|
# Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
|
||||||
|
from(
|
||||||
|
object in Object,
|
||||||
|
where:
|
||||||
|
fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
|
||||||
|
select: %{
|
||||||
|
id: object.id,
|
||||||
|
tag: fragment("(?)->'tag'", object.data)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
|
||||||
|
on: hashtags_objects.object_id == o.id
|
||||||
|
)
|
||||||
|
|> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()}
|
||||||
|
defp transfer_object_hashtags(object) do
|
||||||
|
embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
|
||||||
|
hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
|
||||||
|
|
||||||
|
if Enum.any?(hashtags) do
|
||||||
|
transfer_object_hashtags(object, hashtags)
|
||||||
|
else
|
||||||
|
{:noop, object.id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp transfer_object_hashtags(object, hashtags) do
|
||||||
|
Repo.transaction(fn ->
|
||||||
|
with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
|
||||||
|
maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
|
||||||
|
base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}"
|
||||||
|
|
||||||
|
try do
|
||||||
|
with {rows_count, _} when is_integer(rows_count) <-
|
||||||
|
Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do
|
||||||
|
object.id
|
||||||
|
else
|
||||||
|
e ->
|
||||||
|
Logger.error("#{base_error}: #{inspect(e)}")
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
e ->
|
||||||
|
Logger.error("#{base_error}: #{inspect(e)}")
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
e ->
|
||||||
|
error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
|
||||||
|
Logger.error(error)
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def retry_failed do
|
||||||
|
data_migration_id = data_migration_id()
|
||||||
|
|
||||||
|
failed_objects_query()
|
||||||
|
|> Repo.chunk_stream(100, :one)
|
||||||
|
|> Stream.each(fn object ->
|
||||||
|
with {res, _} when res != :error <- transfer_object_hashtags(object) do
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"DELETE FROM data_migration_failed_ids " <>
|
||||||
|
"WHERE data_migration_id = $1 AND record_id = $2",
|
||||||
|
[data_migration_id, object.id]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> Stream.run()
|
||||||
|
|
||||||
|
put_stat(:failed_count, failures_count())
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
force_continue()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp failed_objects_query do
|
||||||
|
from(o in Object)
|
||||||
|
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
|
||||||
|
on: dmf.record_id == o.id
|
||||||
|
)
|
||||||
|
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|
||||||
|
|> order_by([o], asc: o.id)
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Service func to delete `hashtags_objects` for legacy objects not associated with Create activity.
|
||||||
|
Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`).
|
||||||
|
"""
|
||||||
|
def delete_non_create_activities_hashtags do
|
||||||
|
hashtags_objects_cleanup_query = """
|
||||||
|
DELETE FROM hashtags_objects WHERE object_id IN
|
||||||
|
(SELECT DISTINCT objects.id FROM objects
|
||||||
|
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
|
||||||
|
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
|
||||||
|
(objects.data->>'id')
|
||||||
|
AND activities.data->>'type' = 'Create'
|
||||||
|
WHERE activities.id IS NULL);
|
||||||
|
"""
|
||||||
|
|
||||||
|
hashtags_cleanup_query = """
|
||||||
|
DELETE FROM hashtags WHERE id IN
|
||||||
|
(SELECT hashtags.id FROM hashtags
|
||||||
|
LEFT OUTER JOIN hashtags_objects
|
||||||
|
ON hashtags_objects.hashtag_id = hashtags.id
|
||||||
|
WHERE hashtags_objects.hashtag_id IS NULL);
|
||||||
|
"""
|
||||||
|
|
||||||
|
{:ok, %{num_rows: hashtags_objects_count}} =
|
||||||
|
Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity)
|
||||||
|
|
||||||
|
{:ok, %{num_rows: hashtags_count}} =
|
||||||
|
Repo.query(hashtags_cleanup_query, [], timeout: :infinity)
|
||||||
|
|
||||||
|
{:ok, hashtags_objects_count, hashtags_count}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,210 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.Support.BaseMigrator do
|
||||||
|
@moduledoc """
|
||||||
|
Base background migrator functionality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@callback perform() :: any()
|
||||||
|
@callback retry_failed() :: any()
|
||||||
|
@callback feature_config_path() :: list(atom())
|
||||||
|
@callback query() :: Ecto.Query.t()
|
||||||
|
@callback fault_rate_allowance() :: integer() | float()
|
||||||
|
|
||||||
|
defmacro __using__(_opts) do
|
||||||
|
quote do
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias __MODULE__.State
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
@behaviour Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
|
||||||
|
defdelegate data_migration(), to: State
|
||||||
|
defdelegate data_migration_id(), to: State
|
||||||
|
defdelegate state(), to: State
|
||||||
|
defdelegate persist_state(), to: State, as: :persist_to_db
|
||||||
|
defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key
|
||||||
|
defdelegate put_stat(key, value), to: State, as: :put_data_key
|
||||||
|
defdelegate increment_stat(key, increment), to: State, as: :increment_data_key
|
||||||
|
|
||||||
|
@reg_name {:global, __MODULE__}
|
||||||
|
|
||||||
|
def whereis, do: GenServer.whereis(@reg_name)
|
||||||
|
|
||||||
|
def start_link(_) do
|
||||||
|
case whereis() do
|
||||||
|
nil ->
|
||||||
|
GenServer.start_link(__MODULE__, nil, name: @reg_name)
|
||||||
|
|
||||||
|
pid ->
|
||||||
|
{:ok, pid}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(_) do
|
||||||
|
{:ok, nil, {:continue, :init_state}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_continue(:init_state, _state) do
|
||||||
|
{:ok, _} = State.start_link(nil)
|
||||||
|
|
||||||
|
data_migration = data_migration()
|
||||||
|
manual_migrations = Config.get([:instance, :manual_data_migrations], [])
|
||||||
|
|
||||||
|
cond do
|
||||||
|
Config.get(:env) == :test ->
|
||||||
|
update_status(:noop)
|
||||||
|
|
||||||
|
is_nil(data_migration) ->
|
||||||
|
message = "Data migration does not exist."
|
||||||
|
update_status(:failed, message)
|
||||||
|
Logger.error("#{__MODULE__}: #{message}")
|
||||||
|
|
||||||
|
data_migration.state == :manual or data_migration.name in manual_migrations ->
|
||||||
|
message = "Data migration is in manual execution or manual fix mode."
|
||||||
|
update_status(:manual, message)
|
||||||
|
Logger.warn("#{__MODULE__}: #{message}")
|
||||||
|
|
||||||
|
data_migration.state == :complete ->
|
||||||
|
on_complete(data_migration)
|
||||||
|
|
||||||
|
true ->
|
||||||
|
send(self(), :perform)
|
||||||
|
end
|
||||||
|
|
||||||
|
{:noreply, nil}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info(:perform, state) do
|
||||||
|
State.reinit()
|
||||||
|
|
||||||
|
update_status(:running)
|
||||||
|
put_stat(:iteration_processed_count, 0)
|
||||||
|
put_stat(:started_at, NaiveDateTime.utc_now())
|
||||||
|
|
||||||
|
perform()
|
||||||
|
|
||||||
|
fault_rate = fault_rate()
|
||||||
|
put_stat(:fault_rate, fault_rate)
|
||||||
|
fault_rate_allowance = fault_rate_allowance()
|
||||||
|
|
||||||
|
cond do
|
||||||
|
fault_rate == 0 ->
|
||||||
|
set_complete()
|
||||||
|
|
||||||
|
is_float(fault_rate) and fault_rate <= fault_rate_allowance ->
|
||||||
|
message = """
|
||||||
|
Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}.
|
||||||
|
Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
Logger.warn("#{__MODULE__}: #{message}")
|
||||||
|
update_status(:manual, message)
|
||||||
|
on_complete(data_migration())
|
||||||
|
|
||||||
|
true ->
|
||||||
|
message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`."
|
||||||
|
Logger.error("#{__MODULE__}: #{message}")
|
||||||
|
update_status(:failed, message)
|
||||||
|
end
|
||||||
|
|
||||||
|
persist_state()
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp on_complete(data_migration) do
|
||||||
|
if data_migration.feature_lock || feature_state() == :disabled do
|
||||||
|
Logger.warn(
|
||||||
|
"#{__MODULE__}: migration complete but feature is locked; consider enabling."
|
||||||
|
)
|
||||||
|
|
||||||
|
:noop
|
||||||
|
else
|
||||||
|
Config.put(feature_config_path(), :enabled)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc "Approximate count for current iteration (including processed records count)"
|
||||||
|
def count(force \\ false, timeout \\ :infinity) do
|
||||||
|
stored_count = get_stat(:count)
|
||||||
|
|
||||||
|
if stored_count && !force do
|
||||||
|
stored_count
|
||||||
|
else
|
||||||
|
processed_count = get_stat(:processed_count, 0)
|
||||||
|
max_processed_id = get_stat(:max_processed_id, 0)
|
||||||
|
query = where(query(), [entity], entity.id > ^max_processed_id)
|
||||||
|
|
||||||
|
count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
|
||||||
|
put_stat(:count, count)
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
count
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def failures_count do
|
||||||
|
with {:ok, %{rows: [[count]]}} <-
|
||||||
|
Repo.query(
|
||||||
|
"SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
|
||||||
|
[data_migration_id()]
|
||||||
|
) do
|
||||||
|
count
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def feature_state, do: Config.get(feature_config_path())
|
||||||
|
|
||||||
|
def force_continue do
|
||||||
|
send(whereis(), :perform)
|
||||||
|
end
|
||||||
|
|
||||||
|
def force_restart do
|
||||||
|
:ok = State.reset()
|
||||||
|
force_continue()
|
||||||
|
end
|
||||||
|
|
||||||
|
def set_complete do
|
||||||
|
update_status(:complete)
|
||||||
|
persist_state()
|
||||||
|
on_complete(data_migration())
|
||||||
|
end
|
||||||
|
|
||||||
|
defp update_status(status, message \\ nil) do
|
||||||
|
put_stat(:state, status)
|
||||||
|
put_stat(:message, message)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fault_rate do
|
||||||
|
with failures_count when is_integer(failures_count) <- failures_count() do
|
||||||
|
failures_count / Enum.max([get_stat(:affected_count, 0), 1])
|
||||||
|
else
|
||||||
|
_ -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp records_per_second do
|
||||||
|
get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp running_time do
|
||||||
|
NaiveDateTime.diff(
|
||||||
|
NaiveDateTime.utc_now(),
|
||||||
|
get_stat(:started_at, NaiveDateTime.utc_now())
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,117 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.Support.BaseMigratorState do
|
||||||
|
@moduledoc """
|
||||||
|
Base background migrator state functionality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@callback data_migration() :: Pleroma.DataMigration.t()
|
||||||
|
|
||||||
|
defmacro __using__(_opts) do
|
||||||
|
quote do
|
||||||
|
use Agent
|
||||||
|
|
||||||
|
alias Pleroma.DataMigration
|
||||||
|
|
||||||
|
@behaviour Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
@reg_name {:global, __MODULE__}
|
||||||
|
|
||||||
|
def start_link(_) do
|
||||||
|
Agent.start_link(fn -> load_state_from_db() end, name: @reg_name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def data_migration, do: raise("data_migration/0 is not implemented")
|
||||||
|
defoverridable data_migration: 0
|
||||||
|
|
||||||
|
defp load_state_from_db do
|
||||||
|
data_migration = data_migration()
|
||||||
|
|
||||||
|
data =
|
||||||
|
if data_migration do
|
||||||
|
Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end)
|
||||||
|
else
|
||||||
|
%{}
|
||||||
|
end
|
||||||
|
|
||||||
|
%{
|
||||||
|
data_migration_id: data_migration && data_migration.id,
|
||||||
|
data: data
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def persist_to_db do
|
||||||
|
%{data_migration_id: data_migration_id, data: data} = state()
|
||||||
|
|
||||||
|
if data_migration_id do
|
||||||
|
DataMigration.update_one_by_id(data_migration_id, data: data)
|
||||||
|
else
|
||||||
|
{:error, :nil_data_migration_id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def reset do
|
||||||
|
%{data_migration_id: data_migration_id} = state()
|
||||||
|
|
||||||
|
with false <- is_nil(data_migration_id),
|
||||||
|
:ok <-
|
||||||
|
DataMigration.update_one_by_id(data_migration_id,
|
||||||
|
state: :pending,
|
||||||
|
data: %{}
|
||||||
|
) do
|
||||||
|
reinit()
|
||||||
|
else
|
||||||
|
true -> {:error, :nil_data_migration_id}
|
||||||
|
e -> e
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def reinit do
|
||||||
|
Agent.update(@reg_name, fn _state -> load_state_from_db() end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def state do
|
||||||
|
Agent.get(@reg_name, & &1)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_data_key(key, default \\ nil) do
|
||||||
|
get_in(state(), [:data, key]) || default
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_data_key(key, value) do
|
||||||
|
_ = persist_non_data_change(key, value)
|
||||||
|
|
||||||
|
Agent.update(@reg_name, fn state ->
|
||||||
|
put_in(state, [:data, key], value)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def increment_data_key(key, increment \\ 1) do
|
||||||
|
Agent.update(@reg_name, fn state ->
|
||||||
|
initial_value = get_in(state, [:data, key]) || 0
|
||||||
|
updated_value = initial_value + increment
|
||||||
|
put_in(state, [:data, key], updated_value)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp persist_non_data_change(:state, value) do
|
||||||
|
with true <- get_data_key(:state) != value,
|
||||||
|
true <- value in Pleroma.DataMigration.State.__valid_values__(),
|
||||||
|
%{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <-
|
||||||
|
state() do
|
||||||
|
DataMigration.update_one_by_id(data_migration_id, state: value)
|
||||||
|
else
|
||||||
|
false -> :ok
|
||||||
|
_ -> {:error, :nil_data_migration_id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp persist_non_data_change(_, _) do
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def data_migration_id, do: Map.get(state(), :data_migration_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,45 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
|
||||||
|
@moduledoc """
|
||||||
|
Extracts metadata about the upload, such as width/height
|
||||||
|
"""
|
||||||
|
require Logger
|
||||||
|
|
||||||
|
@behaviour Pleroma.Upload.Filter
|
||||||
|
|
||||||
|
@spec filter(Pleroma.Upload.t()) ::
|
||||||
|
{:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()}
|
||||||
|
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
|
||||||
|
try do
|
||||||
|
image =
|
||||||
|
file
|
||||||
|
|> Mogrify.open()
|
||||||
|
|> Mogrify.verbose()
|
||||||
|
|
||||||
|
upload =
|
||||||
|
upload
|
||||||
|
|> Map.put(:width, image.width)
|
||||||
|
|> Map.put(:height, image.height)
|
||||||
|
|> Map.put(:blurhash, get_blurhash(file))
|
||||||
|
|
||||||
|
{:ok, :filtered, upload}
|
||||||
|
rescue
|
||||||
|
e in ErlangError ->
|
||||||
|
Logger.warn("#{__MODULE__}: #{inspect(e)}")
|
||||||
|
{:ok, :noop}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def filter(_), do: {:ok, :noop}
|
||||||
|
|
||||||
|
defp get_blurhash(file) do
|
||||||
|
with {:ok, blurhash} <- :eblurhash.magick(file) do
|
||||||
|
blurhash
|
||||||
|
else
|
||||||
|
_ -> nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,59 @@
|
|||||||
|
defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
|
||||||
|
@behaviour Pleroma.Web.ActivityPub.MRF
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.User
|
||||||
|
alias Pleroma.Web.CommonAPI
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def filter(message) do
|
||||||
|
with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
|
||||||
|
%User{actor_type: "Service"} = follower <-
|
||||||
|
User.get_cached_by_nickname(follower_nickname),
|
||||||
|
%{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
|
||||||
|
try_follow(follower, message)
|
||||||
|
else
|
||||||
|
nil ->
|
||||||
|
Logger.warn(
|
||||||
|
"#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname
|
||||||
|
account does not exist, or the account is not correctly configured as a bot."
|
||||||
|
)
|
||||||
|
|
||||||
|
{:ok, message}
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp try_follow(follower, message) do
|
||||||
|
to = Map.get(message, "to", [])
|
||||||
|
cc = Map.get(message, "cc", [])
|
||||||
|
actor = [message["actor"]]
|
||||||
|
|
||||||
|
Enum.concat([to, cc, actor])
|
||||||
|
|> List.flatten()
|
||||||
|
|> Enum.uniq()
|
||||||
|
|> User.get_all_by_ap_id()
|
||||||
|
|> Enum.each(fn user ->
|
||||||
|
with false <- user.local,
|
||||||
|
false <- User.following?(follower, user),
|
||||||
|
false <- User.locked?(user),
|
||||||
|
false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do
|
||||||
|
Logger.debug(
|
||||||
|
"#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
|
||||||
|
)
|
||||||
|
|
||||||
|
CommonAPI.follow(follower, user)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def describe do
|
||||||
|
{:ok, %{}}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,116 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
|
||||||
|
require Pleroma.Constants
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Object
|
||||||
|
|
||||||
|
@moduledoc """
|
||||||
|
Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #)
|
||||||
|
|
||||||
|
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@behaviour Pleroma.Web.ActivityPub.MRF
|
||||||
|
|
||||||
|
defp check_reject(message, hashtags) do
|
||||||
|
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
|
||||||
|
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp check_ftl_removal(%{"to" => to} = message, hashtags) do
|
||||||
|
if Pleroma.Constants.as_public() in to and
|
||||||
|
Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
|
||||||
|
match in hashtags
|
||||||
|
end) do
|
||||||
|
to = List.delete(to, Pleroma.Constants.as_public())
|
||||||
|
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
|
||||||
|
|
||||||
|
message =
|
||||||
|
message
|
||||||
|
|> Map.put("to", to)
|
||||||
|
|> Map.put("cc", cc)
|
||||||
|
|> Kernel.put_in(["object", "to"], to)
|
||||||
|
|> Kernel.put_in(["object", "cc"], cc)
|
||||||
|
|
||||||
|
{:ok, message}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
|
||||||
|
|
||||||
|
defp check_sensitive(message, hashtags) do
|
||||||
|
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
|
||||||
|
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def filter(%{"type" => "Create", "object" => object} = message) do
|
||||||
|
hashtags = Object.hashtags(%Object{data: object})
|
||||||
|
|
||||||
|
if hashtags != [] do
|
||||||
|
with {:ok, message} <- check_reject(message, hashtags),
|
||||||
|
{:ok, message} <- check_ftl_removal(message, hashtags),
|
||||||
|
{:ok, message} <- check_sensitive(message, hashtags) do
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def filter(message), do: {:ok, message}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def describe do
|
||||||
|
mrf_hashtag =
|
||||||
|
Config.get(:mrf_hashtag)
|
||||||
|
|> Enum.into(%{})
|
||||||
|
|
||||||
|
{:ok, %{mrf_hashtag: mrf_hashtag}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def config_description do
|
||||||
|
%{
|
||||||
|
key: :mrf_hashtag,
|
||||||
|
related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy",
|
||||||
|
label: "MRF Hashtag",
|
||||||
|
description: @moduledoc,
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :reject,
|
||||||
|
type: {:list, :string},
|
||||||
|
description: "A list of hashtags which result in message being rejected.",
|
||||||
|
suggestions: ["foo"]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :federated_timeline_removal,
|
||||||
|
type: {:list, :string},
|
||||||
|
description:
|
||||||
|
"A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
|
||||||
|
suggestions: ["foo"]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :sensitive,
|
||||||
|
type: {:list, :string},
|
||||||
|
description:
|
||||||
|
"A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
|
||||||
|
suggestions: ["nsfw", "r18"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,77 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
import Ecto.Changeset
|
||||||
|
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
|
||||||
|
|
||||||
|
require Pleroma.Constants
|
||||||
|
|
||||||
|
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||||
|
alias Pleroma.User
|
||||||
|
|
||||||
|
@primary_key false
|
||||||
|
|
||||||
|
embedded_schema do
|
||||||
|
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
||||||
|
field(:target)
|
||||||
|
field(:object, ObjectValidators.ObjectID)
|
||||||
|
field(:actor, ObjectValidators.ObjectID)
|
||||||
|
field(:type)
|
||||||
|
field(:to, ObjectValidators.Recipients, default: [])
|
||||||
|
field(:cc, ObjectValidators.Recipients, default: [])
|
||||||
|
end
|
||||||
|
|
||||||
|
def cast_and_validate(data) do
|
||||||
|
{:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"])
|
||||||
|
|
||||||
|
{:ok, actor} = maybe_refetch_user(actor)
|
||||||
|
|
||||||
|
data
|
||||||
|
|> maybe_fix_data_for_mastodon(actor)
|
||||||
|
|> cast_data()
|
||||||
|
|> validate_data(actor)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp maybe_fix_data_for_mastodon(data, actor) do
|
||||||
|
# Mastodon sends pin/unpin objects without id, to, cc fields
|
||||||
|
data
|
||||||
|
|> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id())
|
||||||
|
|> Map.put_new("to", [Pleroma.Constants.as_public()])
|
||||||
|
|> Map.put_new("cc", [actor.follower_address])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp cast_data(data) do
|
||||||
|
cast(%__MODULE__{}, data, __schema__(:fields))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp validate_data(changeset, actor) do
|
||||||
|
changeset
|
||||||
|
|> validate_required([:id, :target, :object, :actor, :type, :to, :cc])
|
||||||
|
|> validate_inclusion(:type, ~w(Add Remove))
|
||||||
|
|> validate_actor_presence()
|
||||||
|
|> validate_collection_belongs_to_actor(actor)
|
||||||
|
|> validate_object_presence()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp validate_collection_belongs_to_actor(changeset, actor) do
|
||||||
|
validate_change(changeset, :target, fn :target, target ->
|
||||||
|
if target == actor.featured_address do
|
||||||
|
[]
|
||||||
|
else
|
||||||
|
[target: "collection doesn't belong to actor"]
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do
|
||||||
|
{:ok, user}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp maybe_refetch_user(%User{ap_id: ap_id}) do
|
||||||
|
Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id)
|
||||||
|
end
|
||||||
|
end
|
@ -1,29 +0,0 @@
|
|||||||
# Pleroma: A lightweight social networking server
|
|
||||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
|
|
||||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
|
|
||||||
use Ecto.Schema
|
|
||||||
|
|
||||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
|
|
||||||
|
|
||||||
import Ecto.Changeset
|
|
||||||
|
|
||||||
@primary_key false
|
|
||||||
|
|
||||||
embedded_schema do
|
|
||||||
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
|
||||||
field(:actor, ObjectValidators.ObjectID)
|
|
||||||
field(:type, :string)
|
|
||||||
field(:to, ObjectValidators.Recipients, default: [])
|
|
||||||
field(:cc, ObjectValidators.Recipients, default: [])
|
|
||||||
field(:bto, ObjectValidators.Recipients, default: [])
|
|
||||||
field(:bcc, ObjectValidators.Recipients, default: [])
|
|
||||||
embeds_one(:object, NoteValidator)
|
|
||||||
end
|
|
||||||
|
|
||||||
def cast_data(data) do
|
|
||||||
cast(%__MODULE__{}, data, __schema__(:fields))
|
|
||||||
end
|
|
||||||
end
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue