commit
1a45aa1278
@ -0,0 +1,45 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.DataMigration do
|
||||
use Ecto.Schema
|
||||
|
||||
alias Pleroma.DataMigration
|
||||
alias Pleroma.DataMigration.State
|
||||
alias Pleroma.Repo
|
||||
|
||||
import Ecto.Changeset
|
||||
import Ecto.Query
|
||||
|
||||
schema "data_migrations" do
|
||||
field(:name, :string)
|
||||
field(:state, State, default: :pending)
|
||||
field(:feature_lock, :boolean, default: false)
|
||||
field(:params, :map, default: %{})
|
||||
field(:data, :map, default: %{})
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def changeset(data_migration, params \\ %{}) do
|
||||
data_migration
|
||||
|> cast(params, [:name, :state, :feature_lock, :params, :data])
|
||||
|> validate_required([:name])
|
||||
|> unique_constraint(:name)
|
||||
end
|
||||
|
||||
def update_one_by_id(id, params \\ %{}) do
|
||||
with {1, _} <-
|
||||
from(dm in DataMigration, where: dm.id == ^id)
|
||||
|> Repo.update_all(set: params) do
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
def get_by_name(name) do
|
||||
Repo.get_by(DataMigration, name: name)
|
||||
end
|
||||
|
||||
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
|
||||
end
|
@ -1,256 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
#
|
||||
# This file is derived from Earmark, under the following copyright:
|
||||
# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
|
||||
defmodule Pleroma.EarmarkRenderer do
|
||||
@moduledoc false
|
||||
|
||||
alias Earmark.Block
|
||||
alias Earmark.Context
|
||||
alias Earmark.HtmlRenderer
|
||||
alias Earmark.Options
|
||||
|
||||
import Earmark.Inline, only: [convert: 3]
|
||||
import Earmark.Helpers.HtmlHelpers
|
||||
import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
|
||||
import Earmark.Context, only: [append: 2, set_value: 2]
|
||||
import Earmark.Options, only: [get_mapper: 1]
|
||||
|
||||
@doc false
|
||||
def render(blocks, %Context{options: %Options{}} = context) do
|
||||
messages = get_messages(context)
|
||||
|
||||
{contexts, html} =
|
||||
get_mapper(context.options).(
|
||||
blocks,
|
||||
&render_block(&1, put_in(context.options.messages, []))
|
||||
)
|
||||
|> Enum.unzip()
|
||||
|
||||
all_messages =
|
||||
contexts
|
||||
|> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
|
||||
|
||||
{put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
|
||||
end
|
||||
|
||||
#############
|
||||
# Paragraph #
|
||||
#############
|
||||
defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
|
||||
lines = convert(lines, lnb, context)
|
||||
add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
|
||||
end
|
||||
|
||||
########
|
||||
# Html #
|
||||
########
|
||||
defp render_block(%Block.Html{html: html}, context) do
|
||||
{context, html}
|
||||
end
|
||||
|
||||
defp render_block(%Block.HtmlComment{lines: lines}, context) do
|
||||
{context, lines}
|
||||
end
|
||||
|
||||
defp render_block(%Block.HtmlOneline{html: html}, context) do
|
||||
{context, html}
|
||||
end
|
||||
|
||||
#########
|
||||
# Ruler #
|
||||
#########
|
||||
defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
|
||||
add_attrs(context, "<hr />", attrs, [], lnb)
|
||||
end
|
||||
|
||||
###########
|
||||
# Heading #
|
||||
###########
|
||||
defp render_block(
|
||||
%Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
|
||||
context
|
||||
) do
|
||||
converted = convert(content, lnb, context)
|
||||
html = "<h#{level}>#{converted.value}</h#{level}>"
|
||||
add_attrs(converted, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
##############
|
||||
# Blockquote #
|
||||
##############
|
||||
|
||||
defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
|
||||
{context1, body} = render(blocks, context)
|
||||
html = "<blockquote>#{body}</blockquote>"
|
||||
add_attrs(context1, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
#########
|
||||
# Table #
|
||||
#########
|
||||
|
||||
defp render_block(
|
||||
%Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
|
||||
context
|
||||
) do
|
||||
{context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
|
||||
context2 = set_value(context1, html)
|
||||
|
||||
context3 =
|
||||
if header do
|
||||
append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
|
||||
else
|
||||
# Maybe an error, needed append(context, html)
|
||||
context2
|
||||
end
|
||||
|
||||
context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
|
||||
|
||||
{context4, [context4.value, "</table>"]}
|
||||
end
|
||||
|
||||
########
|
||||
# Code #
|
||||
########
|
||||
|
||||
defp render_block(
|
||||
%Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
|
||||
%Context{options: options} = context
|
||||
) do
|
||||
class =
|
||||
if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
|
||||
|
||||
tag = ~s[<pre><code#{class}>]
|
||||
lines = options.render_code.(block)
|
||||
html = ~s[#{tag}#{lines}</code></pre>]
|
||||
add_attrs(context, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
#########
|
||||
# Lists #
|
||||
#########
|
||||
|
||||
defp render_block(
|
||||
%Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
|
||||
context
|
||||
) do
|
||||
{context1, content} = render(items, context)
|
||||
html = "<#{type}#{start}>#{content}</#{type}>"
|
||||
add_attrs(context1, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
# format a single paragraph list item, and remove the para tags
|
||||
defp render_block(
|
||||
%Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
|
||||
context
|
||||
)
|
||||
when length(blocks) == 1 do
|
||||
{context1, content} = render(blocks, context)
|
||||
content = Regex.replace(~r{</?p>}, content, "")
|
||||
html = "<li>#{content}</li>"
|
||||
add_attrs(context1, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
# format a spaced list item
|
||||
defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
|
||||
{context1, content} = render(blocks, context)
|
||||
html = "<li>#{content}</li>"
|
||||
add_attrs(context1, html, attrs, [], lnb)
|
||||
end
|
||||
|
||||
##################
|
||||
# Footnote Block #
|
||||
##################
|
||||
|
||||
defp render_block(%Block.FnList{blocks: footnotes}, context) do
|
||||
items =
|
||||
Enum.map(footnotes, fn note ->
|
||||
blocks = append_footnote_link(note)
|
||||
%Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
|
||||
end)
|
||||
|
||||
{context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
|
||||
{context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
|
||||
end
|
||||
|
||||
#######################################
|
||||
# Isolated IALs are rendered as paras #
|
||||
#######################################
|
||||
|
||||
defp render_block(%Block.Ial{verbatim: verbatim}, context) do
|
||||
{context, "<p>{:#{verbatim}}</p>"}
|
||||
end
|
||||
|
||||
####################
|
||||
# IDDef is ignored #
|
||||
####################
|
||||
|
||||
defp render_block(%Block.IdDef{}, context), do: {context, ""}
|
||||
|
||||
#####################################
|
||||
# And here are the inline renderers #
|
||||
#####################################
|
||||
|
||||
defdelegate br, to: HtmlRenderer
|
||||
defdelegate codespan(text), to: HtmlRenderer
|
||||
defdelegate em(text), to: HtmlRenderer
|
||||
defdelegate strong(text), to: HtmlRenderer
|
||||
defdelegate strikethrough(text), to: HtmlRenderer
|
||||
|
||||
defdelegate link(url, text), to: HtmlRenderer
|
||||
defdelegate link(url, text, title), to: HtmlRenderer
|
||||
|
||||
defdelegate image(path, alt, title), to: HtmlRenderer
|
||||
|
||||
defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
|
||||
|
||||
# Table rows
|
||||
defp add_trs(context, rows, tag, aligns, lnb) do
|
||||
numbered_rows =
|
||||
rows
|
||||
|> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
|
||||
|
||||
numbered_rows
|
||||
|> Enum.reduce(context, fn {row, lnb}, ctx ->
|
||||
append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
|
||||
end)
|
||||
end
|
||||
|
||||
defp add_tds(context, row, tag, aligns, lnb) do
|
||||
Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
|
||||
end
|
||||
|
||||
defp add_td_fn(row, tag, aligns, lnb) do
|
||||
fn n, ctx ->
|
||||
style =
|
||||
case Enum.at(aligns, n - 1, :default) do
|
||||
:default -> ""
|
||||
align -> " style=\"text-align: #{align}\""
|
||||
end
|
||||
|
||||
col = Enum.at(row, n - 1)
|
||||
converted = convert(col, lnb, set_messages(ctx, []))
|
||||
append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
|
||||
end
|
||||
end
|
||||
|
||||
###############################
|
||||
# Append Footnote Return Link #
|
||||
###############################
|
||||
|
||||
defdelegate append_footnote_link(note), to: HtmlRenderer
|
||||
defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
|
||||
|
||||
defdelegate render_code(lines), to: HtmlRenderer
|
||||
|
||||
defp code_classes(language, prefix) do
|
||||
["" | String.split(prefix || "")]
|
||||
|> Enum.map(fn pfx -> "#{pfx}#{language}" end)
|
||||
|> Enum.join(" ")
|
||||
end
|
||||
end
|
@ -0,0 +1,106 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Hashtag do
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
import Ecto.Query
|
||||
|
||||
alias Ecto.Multi
|
||||
alias Pleroma.Hashtag
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.Repo
|
||||
|
||||
schema "hashtags" do
|
||||
field(:name, :string)
|
||||
|
||||
many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def normalize_name(name) do
|
||||
name
|
||||
|> String.downcase()
|
||||
|> String.trim()
|
||||
end
|
||||
|
||||
def get_or_create_by_name(name) do
|
||||
changeset = changeset(%Hashtag{}, %{name: name})
|
||||
|
||||
Repo.insert(
|
||||
changeset,
|
||||
on_conflict: [set: [name: get_field(changeset, :name)]],
|
||||
conflict_target: :name,
|
||||
returning: true
|
||||
)
|
||||
end
|
||||
|
||||
def get_or_create_by_names(names) when is_list(names) do
|
||||
names = Enum.map(names, &normalize_name/1)
|
||||
timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
|
||||
|
||||
structs =
|
||||
Enum.map(names, fn name ->
|
||||
%Hashtag{}
|
||||
|> changeset(%{name: name})
|
||||
|> Map.get(:changes)
|
||||
|> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
|
||||
end)
|
||||
|
||||
try do
|
||||
with {:ok, %{query_op: hashtags}} <-
|
||||
Multi.new()
|
||||
|> Multi.insert_all(:insert_all_op, Hashtag, structs,
|
||||
on_conflict: :nothing,
|
||||
conflict_target: :name
|
||||
)
|
||||
|> Multi.run(:query_op, fn _repo, _changes ->
|
||||
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
|
||||
end)
|
||||
|> Repo.transaction() do
|
||||
{:ok, hashtags}
|
||||
else
|
||||
{:error, _name, value, _changes_so_far} -> {:error, value}
|
||||
end
|
||||
rescue
|
||||
e -> {:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
def changeset(%Hashtag{} = struct, params) do
|
||||
struct
|
||||
|> cast(params, [:name])
|
||||
|> update_change(:name, &normalize_name/1)
|
||||
|> validate_required([:name])
|
||||
|> unique_constraint(:name)
|
||||
end
|
||||
|
||||
def unlink(%Object{id: object_id}) do
|
||||
with {_, hashtag_ids} <-
|
||||
from(hto in "hashtags_objects",
|
||||
where: hto.object_id == ^object_id,
|
||||
select: hto.hashtag_id
|
||||
)
|
||||
|> Repo.delete_all(),
|
||||
{:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
|
||||
{:ok, length(hashtag_ids), unreferenced_count}
|
||||
end
|
||||
end
|
||||
|
||||
@delete_unreferenced_query """
|
||||
DELETE FROM hashtags WHERE id IN
|
||||
(SELECT hashtags.id FROM hashtags
|
||||
LEFT OUTER JOIN hashtags_objects
|
||||
ON hashtags_objects.hashtag_id = hashtags.id
|
||||
WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
|
||||
"""
|
||||
|
||||
def delete_unreferenced(ids) do
|
||||
with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
|
||||
{:ok, deleted_count}
|
||||
end
|
||||
end
|
||||
end
|
@ -0,0 +1,208 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Migrators.HashtagsTableMigrator do
|
||||
defmodule State do
|
||||
use Pleroma.Migrators.Support.BaseMigratorState
|
||||
|
||||
@impl Pleroma.Migrators.Support.BaseMigratorState
|
||||
defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table
|
||||
end
|
||||
|
||||
use Pleroma.Migrators.Support.BaseMigrator
|
||||
|
||||
alias Pleroma.Hashtag
|
||||
alias Pleroma.Migrators.Support.BaseMigrator
|
||||
alias Pleroma.Object
|
||||
|
||||
@impl BaseMigrator
|
||||
def feature_config_path, do: [:features, :improved_hashtag_timeline]
|
||||
|
||||
@impl BaseMigrator
|
||||
def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0)
|
||||
|
||||
@impl BaseMigrator
|
||||
def perform do
|
||||
data_migration_id = data_migration_id()
|
||||
max_processed_id = get_stat(:max_processed_id, 0)
|
||||
|
||||
Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...")
|
||||
|
||||
query()
|
||||
|> where([object], object.id > ^max_processed_id)
|
||||
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|
||||
|> Stream.each(fn objects ->
|
||||
object_ids = Enum.map(objects, & &1.id)
|
||||
|
||||
results = Enum.map(objects, &transfer_object_hashtags(&1))
|
||||
|
||||
failed_ids =
|
||||
results
|
||||
|> Enum.filter(&(elem(&1, 0) == :error))
|
||||
|> Enum.map(&elem(&1, 1))
|
||||
|
||||
# Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags
|
||||
chunk_affected_count =
|
||||
results
|
||||
|> Enum.filter(&(elem(&1, 0) == :ok))
|
||||
|> length()
|
||||
|
||||
for failed_id <- failed_ids do
|
||||
_ =
|
||||
Repo.query(
|
||||
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
|
||||
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
|
||||
[data_migration_id, failed_id]
|
||||
)
|
||||
end
|
||||
|
||||
_ =
|
||||
Repo.query(
|
||||
"DELETE FROM data_migration_failed_ids " <>
|
||||
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
|
||||
[data_migration_id, object_ids -- failed_ids]
|
||||
)
|
||||
|
||||
max_object_id = Enum.at(object_ids, -1)
|
||||
|
||||
put_stat(:max_processed_id, max_object_id)
|
||||
increment_stat(:iteration_processed_count, length(object_ids))
|
||||
increment_stat(:processed_count, length(object_ids))
|
||||
increment_stat(:failed_count, length(failed_ids))
|
||||
increment_stat(:affected_count, chunk_affected_count)
|
||||
put_stat(:records_per_second, records_per_second())
|
||||
persist_state()
|
||||
|
||||
# A quick and dirty approach to controlling the load this background migration imposes
|
||||
sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
|
||||
Process.sleep(sleep_interval)
|
||||
end)
|
||||
|> Stream.run()
|
||||
end
|
||||
|
||||
@impl BaseMigrator
|
||||
def query do
|
||||
# Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
|
||||
# Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
|
||||
from(
|
||||
object in Object,
|
||||
where:
|
||||
fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
|
||||
select: %{
|
||||
id: object.id,
|
||||
tag: fragment("(?)->'tag'", object.data)
|
||||
}
|
||||
)
|
||||
|> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
|
||||
on: hashtags_objects.object_id == o.id
|
||||
)
|
||||
|> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
|
||||
end
|
||||
|
||||
@spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()}
|
||||
defp transfer_object_hashtags(object) do
|
||||
embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
|
||||
hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
|
||||
|
||||
if Enum.any?(hashtags) do
|
||||
transfer_object_hashtags(object, hashtags)
|
||||
else
|
||||
{:noop, object.id}
|
||||
end
|
||||
end
|
||||
|
||||
defp transfer_object_hashtags(object, hashtags) do
|
||||
Repo.transaction(fn ->
|
||||
with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
|
||||
maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
|
||||
base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}"
|
||||
|
||||
try do
|
||||
with {rows_count, _} when is_integer(rows_count) <-
|
||||
Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do
|
||||
object.id
|
||||
else
|
||||
e ->
|
||||
Logger.error("#{base_error}: #{inspect(e)}")
|
||||
Repo.rollback(object.id)
|
||||
end
|
||||
rescue
|
||||
e ->
|
||||
Logger.error("#{base_error}: #{inspect(e)}")
|
||||
Repo.rollback(object.id)
|
||||
end
|
||||
else
|
||||
e ->
|
||||
error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
|
||||
Logger.error(error)
|
||||
Repo.rollback(object.id)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@impl BaseMigrator
|
||||
def retry_failed do
|
||||
data_migration_id = data_migration_id()
|
||||
|
||||
failed_objects_query()
|
||||
|> Repo.chunk_stream(100, :one)
|
||||
|> Stream.each(fn object ->
|
||||
with {res, _} when res != :error <- transfer_object_hashtags(object) do
|
||||
_ =
|
||||
Repo.query(
|
||||
"DELETE FROM data_migration_failed_ids " <>
|
||||
"WHERE data_migration_id = $1 AND record_id = $2",
|
||||
[data_migration_id, object.id]
|
||||
)
|
||||
end
|
||||
end)
|
||||
|> Stream.run()
|
||||
|
||||
put_stat(:failed_count, failures_count())
|
||||
persist_state()
|
||||
|
||||
force_continue()
|
||||
end
|
||||
|
||||
defp failed_objects_query do
|
||||
from(o in Object)
|
||||
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
|
||||
on: dmf.record_id == o.id
|
||||
)
|
||||
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|
||||
|> order_by([o], asc: o.id)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Service func to delete `hashtags_objects` for legacy objects not associated with Create activity.
|
||||
Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`).
|
||||
"""
|
||||
def delete_non_create_activities_hashtags do
|
||||
hashtags_objects_cleanup_query = """
|
||||
DELETE FROM hashtags_objects WHERE object_id IN
|
||||
(SELECT DISTINCT objects.id FROM objects
|
||||
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
|
||||
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
|
||||
(objects.data->>'id')
|
||||
AND activities.data->>'type' = 'Create'
|
||||
WHERE activities.id IS NULL);
|
||||
"""
|
||||
|
||||
hashtags_cleanup_query = """
|
||||
DELETE FROM hashtags WHERE id IN
|
||||
(SELECT hashtags.id FROM hashtags
|
||||
LEFT OUTER JOIN hashtags_objects
|
||||
ON hashtags_objects.hashtag_id = hashtags.id
|
||||
WHERE hashtags_objects.hashtag_id IS NULL);
|
||||
"""
|
||||
|
||||
{:ok, %{num_rows: hashtags_objects_count}} =
|
||||
Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity)
|
||||
|
||||
{:ok, %{num_rows: hashtags_count}} =
|
||||
Repo.query(hashtags_cleanup_query, [], timeout: :infinity)
|
||||
|
||||
{:ok, hashtags_objects_count, hashtags_count}
|
||||
end
|
||||
end
|
@ -0,0 +1,210 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Migrators.Support.BaseMigrator do
|
||||
@moduledoc """
|
||||
Base background migrator functionality.
|
||||
"""
|
||||
|
||||
@callback perform() :: any()
|
||||
@callback retry_failed() :: any()
|
||||
@callback feature_config_path() :: list(atom())
|
||||
@callback query() :: Ecto.Query.t()
|
||||
@callback fault_rate_allowance() :: integer() | float()
|
||||
|
||||
defmacro __using__(_opts) do
|
||||
quote do
|
||||
use GenServer
|
||||
|
||||
require Logger
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias __MODULE__.State
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Repo
|
||||
|
||||
@behaviour Pleroma.Migrators.Support.BaseMigrator
|
||||
|
||||
defdelegate data_migration(), to: State
|
||||
defdelegate data_migration_id(), to: State
|
||||
defdelegate state(), to: State
|
||||
defdelegate persist_state(), to: State, as: :persist_to_db
|
||||
defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key
|
||||
defdelegate put_stat(key, value), to: State, as: :put_data_key
|
||||
defdelegate increment_stat(key, increment), to: State, as: :increment_data_key
|
||||
|
||||
@reg_name {:global, __MODULE__}
|
||||
|
||||
def whereis, do: GenServer.whereis(@reg_name)
|
||||
|
||||
def start_link(_) do
|
||||
case whereis() do
|
||||
nil ->
|
||||
GenServer.start_link(__MODULE__, nil, name: @reg_name)
|
||||
|
||||
pid ->
|
||||
{:ok, pid}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(_) do
|
||||
{:ok, nil, {:continue, :init_state}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_continue(:init_state, _state) do
|
||||
{:ok, _} = State.start_link(nil)
|
||||
|
||||
data_migration = data_migration()
|
||||
manual_migrations = Config.get([:instance, :manual_data_migrations], [])
|
||||
|
||||
cond do
|
||||
Config.get(:env) == :test ->
|
||||
update_status(:noop)
|
||||
|
||||
is_nil(data_migration) ->
|
||||
message = "Data migration does not exist."
|
||||
update_status(:failed, message)
|
||||
Logger.error("#{__MODULE__}: #{message}")
|
||||
|
||||
data_migration.state == :manual or data_migration.name in manual_migrations ->
|
||||
message = "Data migration is in manual execution or manual fix mode."
|
||||
update_status(:manual, message)
|
||||
Logger.warn("#{__MODULE__}: #{message}")
|
||||
|
||||
data_migration.state == :complete ->
|
||||
on_complete(data_migration)
|
||||
|
||||
true ->
|
||||
send(self(), :perform)
|
||||
end
|
||||
|
||||
{:noreply, nil}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:perform, state) do
|
||||
State.reinit()
|
||||
|
||||
update_status(:running)
|
||||
put_stat(:iteration_processed_count, 0)
|
||||
put_stat(:started_at, NaiveDateTime.utc_now())
|
||||
|
||||
perform()
|
||||
|
||||
fault_rate = fault_rate()
|
||||
put_stat(:fault_rate, fault_rate)
|
||||
fault_rate_allowance = fault_rate_allowance()
|
||||
|
||||
cond do
|
||||
fault_rate == 0 ->
|
||||
set_complete()
|
||||
|
||||
is_float(fault_rate) and fault_rate <= fault_rate_allowance ->
|
||||
message = """
|
||||
Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}.
|
||||
Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`.
|
||||
"""
|
||||
|
||||
Logger.warn("#{__MODULE__}: #{message}")
|
||||
update_status(:manual, message)
|
||||
on_complete(data_migration())
|
||||
|
||||
true ->
|
||||
message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`."
|
||||
Logger.error("#{__MODULE__}: #{message}")
|
||||
update_status(:failed, message)
|
||||
end
|
||||
|
||||
persist_state()
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
defp on_complete(data_migration) do
|
||||
if data_migration.feature_lock || feature_state() == :disabled do
|
||||
Logger.warn(
|
||||
"#{__MODULE__}: migration complete but feature is locked; consider enabling."
|
||||
)
|
||||
|
||||
:noop
|
||||
else
|
||||
Config.put(feature_config_path(), :enabled)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
@doc "Approximate count for current iteration (including processed records count)"
|
||||
def count(force \\ false, timeout \\ :infinity) do
|
||||
stored_count = get_stat(:count)
|
||||
|
||||
if stored_count && !force do
|
||||
stored_count
|
||||
else
|
||||
processed_count = get_stat(:processed_count, 0)
|
||||
max_processed_id = get_stat(:max_processed_id, 0)
|
||||
query = where(query(), [entity], entity.id > ^max_processed_id)
|
||||
|
||||
count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
|
||||
put_stat(:count, count)
|
||||
persist_state()
|
||||
|
||||
count
|
||||
end
|
||||
end
|
||||
|
||||
def failures_count do
|
||||
with {:ok, %{rows: [[count]]}} <-
|
||||
Repo.query(
|
||||
"SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
|
||||
[data_migration_id()]
|
||||
) do
|
||||
count
|
||||
end
|
||||
end
|
||||
|
||||
def feature_state, do: Config.get(feature_config_path())
|
||||
|
||||
def force_continue do
|
||||
send(whereis(), :perform)
|
||||
end
|
||||
|
||||
def force_restart do
|
||||
:ok = State.reset()
|
||||
force_continue()
|
||||
end
|
||||
|
||||
def set_complete do
|
||||
update_status(:complete)
|
||||
persist_state()
|
||||
on_complete(data_migration())
|
||||
end
|
||||
|
||||
defp update_status(status, message \\ nil) do
|
||||
put_stat(:state, status)
|
||||
put_stat(:message, message)
|
||||
end
|
||||
|
||||
defp fault_rate do
|
||||
with failures_count when is_integer(failures_count) <- failures_count() do
|
||||
failures_count / Enum.max([get_stat(:affected_count, 0), 1])
|
||||
else
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp records_per_second do
|
||||
get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1])
|
||||
end
|
||||
|
||||
defp running_time do
|
||||
NaiveDateTime.diff(
|
||||
NaiveDateTime.utc_now(),
|
||||
get_stat(:started_at, NaiveDateTime.utc_now())
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
@ -0,0 +1,117 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Migrators.Support.BaseMigratorState do
|
||||
@moduledoc """
|
||||
Base background migrator state functionality.
|
||||
"""
|
||||
|
||||
@callback data_migration() :: Pleroma.DataMigration.t()
|
||||
|
||||
defmacro __using__(_opts) do
|
||||
quote do
|
||||
use Agent
|
||||
|
||||
alias Pleroma.DataMigration
|
||||
|
||||
@behaviour Pleroma.Migrators.Support.BaseMigratorState
|
||||
@reg_name {:global, __MODULE__}
|
||||
|
||||
def start_link(_) do
|
||||
Agent.start_link(fn -> load_state_from_db() end, name: @reg_name)
|
||||
end
|
||||
|
||||
def data_migration, do: raise("data_migration/0 is not implemented")
|
||||
defoverridable data_migration: 0
|
||||
|
||||
defp load_state_from_db do
|
||||
data_migration = data_migration()
|
||||
|
||||
data =
|
||||
if data_migration do
|
||||
Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end)
|
||||
else
|
||||
%{}
|
||||
end
|
||||
|
||||
%{
|
||||
data_migration_id: data_migration && data_migration.id,
|
||||
data: data
|
||||
}
|
||||
end
|
||||
|
||||
def persist_to_db do
|
||||
%{data_migration_id: data_migration_id, data: data} = state()
|
||||
|
||||
if data_migration_id do
|
||||
DataMigration.update_one_by_id(data_migration_id, data: data)
|
||||
else
|
||||
{:error, :nil_data_migration_id}
|
||||
end
|
||||
end
|
||||
|
||||
def reset do
|
||||
%{data_migration_id: data_migration_id} = state()
|
||||
|
||||
with false <- is_nil(data_migration_id),
|
||||
:ok <-
|
||||
DataMigration.update_one_by_id(data_migration_id,
|
||||
state: :pending,
|
||||
data: %{}
|
||||
) do
|
||||
reinit()
|
||||
else
|
||||
true -> {:error, :nil_data_migration_id}
|
||||
e -> e
|
||||
end
|
||||
end
|
||||
|
||||
def reinit do
|
||||
Agent.update(@reg_name, fn _state -> load_state_from_db() end)
|
||||
end
|
||||
|
||||
def state do
|
||||
Agent.get(@reg_name, & &1)
|
||||
end
|
||||
|
||||
def get_data_key(key, default \\ nil) do
|
||||
get_in(state(), [:data, key]) || default
|
||||
end
|
||||
|
||||
def put_data_key(key, value) do
|
||||
_ = persist_non_data_change(key, value)
|
||||
|
||||
Agent.update(@reg_name, fn state ->
|
||||
put_in(state, [:data, key], value)
|
||||
end)
|
||||
end
|
||||
|
||||
def increment_data_key(key, increment \\ 1) do
|
||||
Agent.update(@reg_name, fn state ->
|
||||
initial_value = get_in(state, [:data, key]) || 0
|
||||
updated_value = initial_value + increment
|
||||
put_in(state, [:data, key], updated_value)
|
||||
end)
|
||||
end
|
||||
|
||||
defp persist_non_data_change(:state, value) do
|
||||
with true <- get_data_key(:state) != value,
|
||||
true <- value in Pleroma.DataMigration.State.__valid_values__(),
|
||||
%{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <-
|
||||
state() do
|
||||
DataMigration.update_one_by_id(data_migration_id, state: value)
|
||||
else
|
||||
false -> :ok
|
||||
_ -> {:error, :nil_data_migration_id}
|
||||
end
|
||||
end
|
||||
|
||||
defp persist_non_data_change(_, _) do
|
||||
nil
|
||||
end
|
||||
|
||||
def data_migration_id, do: Map.get(state(), :data_migration_id)
|
||||
end
|
||||
end
|
||||
end
|
@ -0,0 +1,59 @@
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web.CommonAPI
|
||||
|
||||
require Logger
|
||||
|
||||
@impl true
|
||||
def filter(message) do
|
||||
with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
|
||||
%User{actor_type: "Service"} = follower <-
|
||||
User.get_cached_by_nickname(follower_nickname),
|
||||
%{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
|
||||
try_follow(follower, message)
|
||||
else
|
||||
nil ->
|
||||
Logger.warn(
|
||||
"#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname
|
||||
account does not exist, or the account is not correctly configured as a bot."
|
||||
)
|
||||
|
||||
{:ok, message}
|
||||
|
||||
_ ->
|
||||
{:ok, message}
|
||||
end
|
||||
end
|
||||
|
||||
defp try_follow(follower, message) do
|
||||
to = Map.get(message, "to", [])
|
||||
cc = Map.get(message, "cc", [])
|
||||
actor = [message["actor"]]
|
||||
|
||||
Enum.concat([to, cc, actor])
|
||||
|> List.flatten()
|
||||
|> Enum.uniq()
|
||||
|> User.get_all_by_ap_id()
|
||||
|> Enum.each(fn user ->
|
||||
with false <- user.local,
|
||||
false <- User.following?(follower, user),
|
||||
false <- User.locked?(user),
|
||||
false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do
|
||||
Logger.debug(
|
||||
"#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
|
||||
)
|
||||
|
||||
CommonAPI.follow(follower, user)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, message}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def describe do
|
||||
{:ok, %{}}
|
||||
end
|
||||
end
|
@ -0,0 +1,116 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Object
|
||||
|
||||
@moduledoc """
|
||||
Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #)
|
||||
|
||||
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
|
||||
"""
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF
|
||||
|
||||
defp check_reject(message, hashtags) do
|
||||
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
|
||||
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
|
||||
else
|
||||
{:ok, message}
|
||||
end
|
||||
end
|
||||
|
||||
defp check_ftl_removal(%{"to" => to} = message, hashtags) do
|
||||
if Pleroma.Constants.as_public() in to and
|
||||
Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
|
||||
match in hashtags
|
||||
end) do
|
||||
to = List.delete(to, Pleroma.Constants.as_public())
|
||||
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
|
||||
|
||||
message =
|
||||
message
|
||||
|> Map.put("to", to)
|
||||
|> Map.put("cc", cc)
|
||||
|> Kernel.put_in(["object", "to"], to)
|
||||
|> Kernel.put_in(["object", "cc"], cc)
|
||||
|
||||
{:ok, message}
|
||||
else
|
||||
{:ok, message}
|
||||
end
|
||||
end
|
||||
|
||||
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
|
||||
|
||||
defp check_sensitive(message, hashtags) do
|
||||
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
|
||||
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
|
||||
else
|
||||
{:ok, message}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def filter(%{"type" => "Create", "object" => object} = message) do
|
||||
hashtags = Object.hashtags(%Object{data: object})
|
||||
|
||||
if hashtags != [] do
|
||||
with {:ok, message} <- check_reject(message, hashtags),
|
||||
{:ok, message} <- check_ftl_removal(message, hashtags),
|
||||
{:ok, message} <- check_sensitive(message, hashtags) do
|
||||
{:ok, message}
|
||||
end
|
||||
else
|
||||
{:ok, message}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def filter(message), do: {:ok, message}
|
||||
|
||||
@impl true
|
||||
def describe do
|
||||
mrf_hashtag =
|
||||
Config.get(:mrf_hashtag)
|
||||
|> Enum.into(%{})
|
||||
|
||||
{:ok, %{mrf_hashtag: mrf_hashtag}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def config_description do
|
||||
%{
|
||||
key: :mrf_hashtag,
|
||||
related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy",
|
||||
label: "MRF Hashtag",
|
||||
description: @moduledoc,
|
||||
children: [
|
||||
%{
|
||||
key: :reject,
|
||||
type: {:list, :string},
|
||||
description: "A list of hashtags which result in message being rejected.",
|
||||
suggestions: ["foo"]
|
||||
},
|
||||
%{
|
||||
key: :federated_timeline_removal,
|
||||
type: {:list, :string},
|
||||
description:
|
||||
"A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
|
||||
suggestions: ["foo"]
|
||||
},
|
||||
%{
|
||||
key: :sensitive,
|
||||
type: {:list, :string},
|
||||
description:
|
||||
"A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
|
||||
suggestions: ["nsfw", "r18"]
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
end
|
@ -0,0 +1,77 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||
alias Pleroma.User
|
||||
|
||||
@primary_key false
|
||||
|
||||
embedded_schema do
|
||||
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
||||
field(:target)
|
||||
field(:object, ObjectValidators.ObjectID)
|
||||
field(:actor, ObjectValidators.ObjectID)
|
||||
field(:type)
|
||||
field(:to, ObjectValidators.Recipients, default: [])
|
||||
field(:cc, ObjectValidators.Recipients, default: [])
|
||||
end
|
||||
|
||||
def cast_and_validate(data) do
|
||||
{:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"])
|
||||
|
||||
{:ok, actor} = maybe_refetch_user(actor)
|
||||
|
||||
data
|
||||
|> maybe_fix_data_for_mastodon(actor)
|
||||
|> cast_data()
|
||||
|> validate_data(actor)
|
||||
end
|
||||
|
||||
defp maybe_fix_data_for_mastodon(data, actor) do
|
||||
# Mastodon sends pin/unpin objects without id, to, cc fields
|
||||
data
|
||||
|> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id())
|
||||
|> Map.put_new("to", [Pleroma.Constants.as_public()])
|
||||
|> Map.put_new("cc", [actor.follower_address])
|
||||
end
|
||||
|
||||
defp cast_data(data) do
|
||||
cast(%__MODULE__{}, data, __schema__(:fields))
|
||||
end
|
||||
|
||||
defp validate_data(changeset, actor) do
|
||||
changeset
|
||||
|> validate_required([:id, :target, :object, :actor, :type, :to, :cc])
|
||||
|> validate_inclusion(:type, ~w(Add Remove))
|
||||
|> validate_actor_presence()
|
||||
|> validate_collection_belongs_to_actor(actor)
|
||||
|> validate_object_presence()
|
||||
end
|
||||
|
||||
defp validate_collection_belongs_to_actor(changeset, actor) do
|
||||
validate_change(changeset, :target, fn :target, target ->
|
||||
if target == actor.featured_address do
|
||||
[]
|
||||
else
|
||||
[target: "collection doesn't belong to actor"]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do
|
||||
{:ok, user}
|
||||
end
|
||||
|
||||
defp maybe_refetch_user(%User{ap_id: ap_id}) do
|
||||
Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id)
|
||||
end
|
||||
end
|
@ -0,0 +1,77 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
|
||||
use Ecto.Schema
|
||||
|
||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key false
|
||||
embedded_schema do
|
||||
# Common
|
||||
field(:type, :string)
|
||||
field(:name, :string)
|
||||
|
||||
# Mention, Hashtag
|
||||
field(:href, ObjectValidators.Uri)
|
||||
|
||||
# Emoji
|
||||
embeds_one :icon, IconObjectValidator, primary_key: false do
|
||||
field(:type, :string)
|
||||
field(:url, ObjectValidators.Uri)
|
||||
end
|
||||
|
||||
field(:updated, ObjectValidators.DateTime)
|
||||
field(:id, ObjectValidators.Uri)
|
||||
end
|
||||
|
||||
def cast_and_validate(data) do
|
||||
data
|
||||
|> cast_data()
|
||||
end
|
||||
|
||||
def cast_data(data) do
|
||||
%__MODULE__{}
|
||||
|> changeset(data)
|
||||
end
|
||||
|
||||
def changeset(struct, %{"type" => "Mention"} = data) do
|
||||
struct
|
||||
|> cast(data, [:type, :name, :href])
|
||||
|> validate_required([:type, :href])
|
||||
end
|
||||
|
||||
def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do
|
||||
name =
|
||||
cond do
|
||||
"#" <> name -> name
|
||||
name -> name
|
||||
end
|
||||
|> String.downcase()
|
||||
|
||||
data = Map.put(data, "name", name)
|
||||
|
||||
struct
|
||||
|> cast(data, [:type, :name, :href])
|
||||
|> validate_required([:type, :name])
|
||||
end
|
||||
|
||||
def changeset(struct, %{"type" => "Emoji"} = data) do
|
||||
data = Map.put(data, "name", String.trim(data["name"], ":"))
|
||||
|
||||
struct
|
||||
|> cast(data, [:type, :name, :updated, :id])
|
||||
|> cast_embed(:icon, with: &icon_changeset/2)
|
||||
|> validate_required([:type, :name, :icon])
|
||||
end
|
||||
|
||||
def icon_changeset(struct, data) do
|
||||
struct
|
||||
|> cast(data, [:type, :url])
|
||||
|> validate_inclusion(:type, ~w[Image])
|
||||
|> validate_required([:type, :url])
|
||||
end
|
||||
end
|
@ -0,0 +1,389 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do
|
||||
alias OpenApiSpex.Operation
|
||||
alias OpenApiSpex.Schema
|
||||
alias Pleroma.Web.ApiSpec.Schemas.ActorType
|
||||
alias Pleroma.Web.ApiSpec.Schemas.ApiError
|
||||
|
||||
import Pleroma.Web.ApiSpec.Helpers
|
||||
|
||||
def open_api_operation(action) do
|
||||
operation = String.to_existing_atom("#{action}_operation")
|
||||
apply(__MODULE__, operation, [])
|
||||
end
|
||||
|
||||
def index_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "List users",
|
||||
operationId: "AdminAPI.UserController.index",
|
||||
security: [%{"oAuth" => ["admin:read:accounts"]}],
|
||||
parameters: [
|
||||
Operation.parameter(:filters, :query, :string, "Comma separated list of filters"),
|
||||
Operation.parameter(:query, :query, :string, "Search users query"),
|
||||
Operation.parameter(:name, :query, :string, "Search by display name"),
|
||||
Operation.parameter(:email, :query, :string, "Search by email"),
|
||||
Operation.parameter(:page, :query, :integer, "Page Number"),
|
||||
Operation.parameter(:page_size, :query, :integer, "Number of users to return per page"),
|
||||
Operation.parameter(
|
||||
:actor_types,
|
||||
:query,
|
||||
%Schema{type: :array, items: ActorType},
|
||||
"Filter by actor type"
|
||||
),
|
||||
Operation.parameter(
|
||||
:tags,
|
||||
:query,
|
||||
%Schema{type: :array, items: %Schema{type: :string}},
|
||||
"Filter by tags"
|
||||
)
|
||||
| admin_api_params()
|
||||
],
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response(
|
||||
"Response",
|
||||
"application/json",
|
||||
%Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
users: %Schema{type: :array, items: user()},
|
||||
count: %Schema{type: :integer},
|
||||
page_size: %Schema{type: :integer}
|
||||
}
|
||||
}
|
||||
),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def create_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Create a single or multiple users",
|
||||
operationId: "AdminAPI.UserController.create",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
description: "POST body for creating users",
|
||||
type: :object,
|
||||
properties: %{
|
||||
users: %Schema{
|
||||
type: :array,
|
||||
items: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
nickname: %Schema{type: :string},
|
||||
email: %Schema{type: :string},
|
||||
password: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response("Response", "application/json", %Schema{
|
||||
type: :array,
|
||||
items: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
code: %Schema{type: :integer},
|
||||
type: %Schema{type: :string},
|
||||
data: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
email: %Schema{type: :string, format: :email},
|
||||
nickname: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError),
|
||||
409 =>
|
||||
Operation.response("Conflict", "application/json", %Schema{
|
||||
type: :array,
|
||||
items: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
code: %Schema{type: :integer},
|
||||
error: %Schema{type: :string},
|
||||
type: %Schema{type: :string},
|
||||
data: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
email: %Schema{type: :string, format: :email},
|
||||
nickname: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def show_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Show user",
|
||||
operationId: "AdminAPI.UserController.show",
|
||||
security: [%{"oAuth" => ["admin:read:accounts"]}],
|
||||
parameters: [
|
||||
Operation.parameter(
|
||||
:nickname,
|
||||
:path,
|
||||
:string,
|
||||
"User nickname or ID"
|
||||
)
|
||||
| admin_api_params()
|
||||
],
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", user()),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError),
|
||||
404 => Operation.response("Not Found", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def follow_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Follow",
|
||||
operationId: "AdminAPI.UserController.follow",
|
||||
security: [%{"oAuth" => ["admin:write:follows"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
follower: %Schema{type: :string, description: "Follower nickname"},
|
||||
followed: %Schema{type: :string, description: "Followed nickname"}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", %Schema{type: :string}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def unfollow_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Unfollow",
|
||||
operationId: "AdminAPI.UserController.unfollow",
|
||||
security: [%{"oAuth" => ["admin:write:follows"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
follower: %Schema{type: :string, description: "Follower nickname"},
|
||||
followed: %Schema{type: :string, description: "Followed nickname"}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", %Schema{type: :string}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def approve_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Approve multiple users",
|
||||
operationId: "AdminAPI.UserController.approve",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
description: "POST body for deleting multiple users",
|
||||
type: :object,
|
||||
properties: %{
|
||||
nicknames: %Schema{
|
||||
type: :array,
|
||||
items: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response("Response", "application/json", %Schema{
|
||||
type: :object,
|
||||
properties: %{user: %Schema{type: :array, items: user()}}
|
||||
}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def toggle_activation_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Toggle user activation",
|
||||
operationId: "AdminAPI.UserController.toggle_activation",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: [
|
||||
Operation.parameter(:nickname, :path, :string, "User nickname")
|
||||
| admin_api_params()
|
||||
],
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", user()),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def activate_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Activate multiple users",
|
||||
operationId: "AdminAPI.UserController.activate",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
description: "POST body for deleting multiple users",
|
||||
type: :object,
|
||||
properties: %{
|
||||
nicknames: %Schema{
|
||||
type: :array,
|
||||
items: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response("Response", "application/json", %Schema{
|
||||
type: :object,
|
||||
properties: %{user: %Schema{type: :array, items: user()}}
|
||||
}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def deactivate_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Deactivates multiple users",
|
||||
operationId: "AdminAPI.UserController.deactivate",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: admin_api_params(),
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
description: "POST body for deleting multiple users",
|
||||
type: :object,
|
||||
properties: %{
|
||||
nicknames: %Schema{
|
||||
type: :array,
|
||||
items: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response("Response", "application/json", %Schema{
|
||||
type: :object,
|
||||
properties: %{user: %Schema{type: :array, items: user()}}
|
||||
}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def delete_operation do
|
||||
%Operation{
|
||||
tags: ["User administration"],
|
||||
summary: "Removes a single or multiple users",
|
||||
operationId: "AdminAPI.UserController.delete",
|
||||
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||
parameters: [
|
||||
Operation.parameter(
|
||||
:nickname,
|
||||
:query,
|
||||
:string,
|
||||
"User nickname"
|
||||
)
|
||||
| admin_api_params()
|
||||
],
|
||||
requestBody:
|
||||
request_body(
|
||||
"Parameters",
|
||||
%Schema{
|
||||
description: "POST body for deleting multiple users",
|
||||
type: :object,
|
||||
properties: %{
|
||||
nicknames: %Schema{
|
||||
type: :array,
|
||||
items: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
responses: %{
|
||||
200 =>
|
||||
Operation.response("Response", "application/json", %Schema{
|
||||
description: "Array of nicknames",
|
||||
type: :array,
|
||||
items: %Schema{type: :string}
|
||||
}),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp user do
|
||||
%Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
id: %Schema{type: :string},
|
||||
email: %Schema{type: :string, format: :email},
|
||||
avatar: %Schema{type: :string, format: :uri},
|
||||
nickname: %Schema{type: :string},
|
||||
display_name: %Schema{type: :string},
|
||||
is_active: %Schema{type: :boolean},
|
||||
local: %Schema{type: :boolean},
|
||||
roles: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
admin: %Schema{type: :boolean},
|
||||
moderator: %Schema{type: :boolean}
|
||||
}
|
||||
},
|
||||
tags: %Schema{type: :array, items: %Schema{type: :string}},
|
||||
is_confirmed: %Schema{type: :boolean},
|
||||
is_approved: %Schema{type: :boolean},
|
||||
url: %Schema{type: :string, format: :uri},
|
||||
registration_reason: %Schema{type: :string, nullable: true},
|
||||
actor_type: %Schema{type: :string}
|
||||
}
|
||||
}
|
||||
end
|
||||
end
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue