Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop
commit
5f62b55a6f
@ -0,0 +1,45 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.DataMigration do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
alias Pleroma.DataMigration
|
||||||
|
alias Pleroma.DataMigration.State
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
import Ecto.Changeset
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
schema "data_migrations" do
|
||||||
|
field(:name, :string)
|
||||||
|
field(:state, State, default: :pending)
|
||||||
|
field(:feature_lock, :boolean, default: false)
|
||||||
|
field(:params, :map, default: %{})
|
||||||
|
field(:data, :map, default: %{})
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
def changeset(data_migration, params \\ %{}) do
|
||||||
|
data_migration
|
||||||
|
|> cast(params, [:name, :state, :feature_lock, :params, :data])
|
||||||
|
|> validate_required([:name])
|
||||||
|
|> unique_constraint(:name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_one_by_id(id, params \\ %{}) do
|
||||||
|
with {1, _} <-
|
||||||
|
from(dm in DataMigration, where: dm.id == ^id)
|
||||||
|
|> Repo.update_all(set: params) do
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_by_name(name) do
|
||||||
|
Repo.get_by(DataMigration, name: name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
|
||||||
|
end
|
@ -0,0 +1,106 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Hashtag do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
import Ecto.Changeset
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias Ecto.Multi
|
||||||
|
alias Pleroma.Hashtag
|
||||||
|
alias Pleroma.Object
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
schema "hashtags" do
|
||||||
|
field(:name, :string)
|
||||||
|
|
||||||
|
many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
def normalize_name(name) do
|
||||||
|
name
|
||||||
|
|> String.downcase()
|
||||||
|
|> String.trim()
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_or_create_by_name(name) do
|
||||||
|
changeset = changeset(%Hashtag{}, %{name: name})
|
||||||
|
|
||||||
|
Repo.insert(
|
||||||
|
changeset,
|
||||||
|
on_conflict: [set: [name: get_field(changeset, :name)]],
|
||||||
|
conflict_target: :name,
|
||||||
|
returning: true
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_or_create_by_names(names) when is_list(names) do
|
||||||
|
names = Enum.map(names, &normalize_name/1)
|
||||||
|
timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
|
||||||
|
|
||||||
|
structs =
|
||||||
|
Enum.map(names, fn name ->
|
||||||
|
%Hashtag{}
|
||||||
|
|> changeset(%{name: name})
|
||||||
|
|> Map.get(:changes)
|
||||||
|
|> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
|
||||||
|
end)
|
||||||
|
|
||||||
|
try do
|
||||||
|
with {:ok, %{query_op: hashtags}} <-
|
||||||
|
Multi.new()
|
||||||
|
|> Multi.insert_all(:insert_all_op, Hashtag, structs,
|
||||||
|
on_conflict: :nothing,
|
||||||
|
conflict_target: :name
|
||||||
|
)
|
||||||
|
|> Multi.run(:query_op, fn _repo, _changes ->
|
||||||
|
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
|
||||||
|
end)
|
||||||
|
|> Repo.transaction() do
|
||||||
|
{:ok, hashtags}
|
||||||
|
else
|
||||||
|
{:error, _name, value, _changes_so_far} -> {:error, value}
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
e -> {:error, e}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def changeset(%Hashtag{} = struct, params) do
|
||||||
|
struct
|
||||||
|
|> cast(params, [:name])
|
||||||
|
|> update_change(:name, &normalize_name/1)
|
||||||
|
|> validate_required([:name])
|
||||||
|
|> unique_constraint(:name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def unlink(%Object{id: object_id}) do
|
||||||
|
with {_, hashtag_ids} <-
|
||||||
|
from(hto in "hashtags_objects",
|
||||||
|
where: hto.object_id == ^object_id,
|
||||||
|
select: hto.hashtag_id
|
||||||
|
)
|
||||||
|
|> Repo.delete_all(),
|
||||||
|
{:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
|
||||||
|
{:ok, length(hashtag_ids), unreferenced_count}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@delete_unreferenced_query """
|
||||||
|
DELETE FROM hashtags WHERE id IN
|
||||||
|
(SELECT hashtags.id FROM hashtags
|
||||||
|
LEFT OUTER JOIN hashtags_objects
|
||||||
|
ON hashtags_objects.hashtag_id = hashtags.id
|
||||||
|
WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
|
||||||
|
"""
|
||||||
|
|
||||||
|
def delete_unreferenced(ids) do
|
||||||
|
with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
|
||||||
|
{:ok, deleted_count}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,208 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.HashtagsTableMigrator do
|
||||||
|
defmodule State do
|
||||||
|
use Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
|
||||||
|
@impl Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table
|
||||||
|
end
|
||||||
|
|
||||||
|
use Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
|
||||||
|
alias Pleroma.Hashtag
|
||||||
|
alias Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
alias Pleroma.Object
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def feature_config_path, do: [:features, :improved_hashtag_timeline]
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0)
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def perform do
|
||||||
|
data_migration_id = data_migration_id()
|
||||||
|
max_processed_id = get_stat(:max_processed_id, 0)
|
||||||
|
|
||||||
|
Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...")
|
||||||
|
|
||||||
|
query()
|
||||||
|
|> where([object], object.id > ^max_processed_id)
|
||||||
|
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|
||||||
|
|> Stream.each(fn objects ->
|
||||||
|
object_ids = Enum.map(objects, & &1.id)
|
||||||
|
|
||||||
|
results = Enum.map(objects, &transfer_object_hashtags(&1))
|
||||||
|
|
||||||
|
failed_ids =
|
||||||
|
results
|
||||||
|
|> Enum.filter(&(elem(&1, 0) == :error))
|
||||||
|
|> Enum.map(&elem(&1, 1))
|
||||||
|
|
||||||
|
# Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags
|
||||||
|
chunk_affected_count =
|
||||||
|
results
|
||||||
|
|> Enum.filter(&(elem(&1, 0) == :ok))
|
||||||
|
|> length()
|
||||||
|
|
||||||
|
for failed_id <- failed_ids do
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
|
||||||
|
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
|
||||||
|
[data_migration_id, failed_id]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"DELETE FROM data_migration_failed_ids " <>
|
||||||
|
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
|
||||||
|
[data_migration_id, object_ids -- failed_ids]
|
||||||
|
)
|
||||||
|
|
||||||
|
max_object_id = Enum.at(object_ids, -1)
|
||||||
|
|
||||||
|
put_stat(:max_processed_id, max_object_id)
|
||||||
|
increment_stat(:iteration_processed_count, length(object_ids))
|
||||||
|
increment_stat(:processed_count, length(object_ids))
|
||||||
|
increment_stat(:failed_count, length(failed_ids))
|
||||||
|
increment_stat(:affected_count, chunk_affected_count)
|
||||||
|
put_stat(:records_per_second, records_per_second())
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
# A quick and dirty approach to controlling the load this background migration imposes
|
||||||
|
sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
|
||||||
|
Process.sleep(sleep_interval)
|
||||||
|
end)
|
||||||
|
|> Stream.run()
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def query do
|
||||||
|
# Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
|
||||||
|
# Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
|
||||||
|
from(
|
||||||
|
object in Object,
|
||||||
|
where:
|
||||||
|
fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
|
||||||
|
select: %{
|
||||||
|
id: object.id,
|
||||||
|
tag: fragment("(?)->'tag'", object.data)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
|
||||||
|
on: hashtags_objects.object_id == o.id
|
||||||
|
)
|
||||||
|
|> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()}
|
||||||
|
defp transfer_object_hashtags(object) do
|
||||||
|
embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
|
||||||
|
hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
|
||||||
|
|
||||||
|
if Enum.any?(hashtags) do
|
||||||
|
transfer_object_hashtags(object, hashtags)
|
||||||
|
else
|
||||||
|
{:noop, object.id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp transfer_object_hashtags(object, hashtags) do
|
||||||
|
Repo.transaction(fn ->
|
||||||
|
with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
|
||||||
|
maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
|
||||||
|
base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}"
|
||||||
|
|
||||||
|
try do
|
||||||
|
with {rows_count, _} when is_integer(rows_count) <-
|
||||||
|
Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do
|
||||||
|
object.id
|
||||||
|
else
|
||||||
|
e ->
|
||||||
|
Logger.error("#{base_error}: #{inspect(e)}")
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
e ->
|
||||||
|
Logger.error("#{base_error}: #{inspect(e)}")
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
e ->
|
||||||
|
error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
|
||||||
|
Logger.error(error)
|
||||||
|
Repo.rollback(object.id)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl BaseMigrator
|
||||||
|
def retry_failed do
|
||||||
|
data_migration_id = data_migration_id()
|
||||||
|
|
||||||
|
failed_objects_query()
|
||||||
|
|> Repo.chunk_stream(100, :one)
|
||||||
|
|> Stream.each(fn object ->
|
||||||
|
with {res, _} when res != :error <- transfer_object_hashtags(object) do
|
||||||
|
_ =
|
||||||
|
Repo.query(
|
||||||
|
"DELETE FROM data_migration_failed_ids " <>
|
||||||
|
"WHERE data_migration_id = $1 AND record_id = $2",
|
||||||
|
[data_migration_id, object.id]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> Stream.run()
|
||||||
|
|
||||||
|
put_stat(:failed_count, failures_count())
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
force_continue()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp failed_objects_query do
|
||||||
|
from(o in Object)
|
||||||
|
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
|
||||||
|
on: dmf.record_id == o.id
|
||||||
|
)
|
||||||
|
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|
||||||
|
|> order_by([o], asc: o.id)
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Service func to delete `hashtags_objects` for legacy objects not associated with Create activity.
|
||||||
|
Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`).
|
||||||
|
"""
|
||||||
|
def delete_non_create_activities_hashtags do
|
||||||
|
hashtags_objects_cleanup_query = """
|
||||||
|
DELETE FROM hashtags_objects WHERE object_id IN
|
||||||
|
(SELECT DISTINCT objects.id FROM objects
|
||||||
|
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
|
||||||
|
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
|
||||||
|
(objects.data->>'id')
|
||||||
|
AND activities.data->>'type' = 'Create'
|
||||||
|
WHERE activities.id IS NULL);
|
||||||
|
"""
|
||||||
|
|
||||||
|
hashtags_cleanup_query = """
|
||||||
|
DELETE FROM hashtags WHERE id IN
|
||||||
|
(SELECT hashtags.id FROM hashtags
|
||||||
|
LEFT OUTER JOIN hashtags_objects
|
||||||
|
ON hashtags_objects.hashtag_id = hashtags.id
|
||||||
|
WHERE hashtags_objects.hashtag_id IS NULL);
|
||||||
|
"""
|
||||||
|
|
||||||
|
{:ok, %{num_rows: hashtags_objects_count}} =
|
||||||
|
Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity)
|
||||||
|
|
||||||
|
{:ok, %{num_rows: hashtags_count}} =
|
||||||
|
Repo.query(hashtags_cleanup_query, [], timeout: :infinity)
|
||||||
|
|
||||||
|
{:ok, hashtags_objects_count, hashtags_count}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,210 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.Support.BaseMigrator do
|
||||||
|
@moduledoc """
|
||||||
|
Base background migrator functionality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@callback perform() :: any()
|
||||||
|
@callback retry_failed() :: any()
|
||||||
|
@callback feature_config_path() :: list(atom())
|
||||||
|
@callback query() :: Ecto.Query.t()
|
||||||
|
@callback fault_rate_allowance() :: integer() | float()
|
||||||
|
|
||||||
|
defmacro __using__(_opts) do
|
||||||
|
quote do
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias __MODULE__.State
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Repo
|
||||||
|
|
||||||
|
@behaviour Pleroma.Migrators.Support.BaseMigrator
|
||||||
|
|
||||||
|
defdelegate data_migration(), to: State
|
||||||
|
defdelegate data_migration_id(), to: State
|
||||||
|
defdelegate state(), to: State
|
||||||
|
defdelegate persist_state(), to: State, as: :persist_to_db
|
||||||
|
defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key
|
||||||
|
defdelegate put_stat(key, value), to: State, as: :put_data_key
|
||||||
|
defdelegate increment_stat(key, increment), to: State, as: :increment_data_key
|
||||||
|
|
||||||
|
@reg_name {:global, __MODULE__}
|
||||||
|
|
||||||
|
def whereis, do: GenServer.whereis(@reg_name)
|
||||||
|
|
||||||
|
def start_link(_) do
|
||||||
|
case whereis() do
|
||||||
|
nil ->
|
||||||
|
GenServer.start_link(__MODULE__, nil, name: @reg_name)
|
||||||
|
|
||||||
|
pid ->
|
||||||
|
{:ok, pid}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(_) do
|
||||||
|
{:ok, nil, {:continue, :init_state}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_continue(:init_state, _state) do
|
||||||
|
{:ok, _} = State.start_link(nil)
|
||||||
|
|
||||||
|
data_migration = data_migration()
|
||||||
|
manual_migrations = Config.get([:instance, :manual_data_migrations], [])
|
||||||
|
|
||||||
|
cond do
|
||||||
|
Config.get(:env) == :test ->
|
||||||
|
update_status(:noop)
|
||||||
|
|
||||||
|
is_nil(data_migration) ->
|
||||||
|
message = "Data migration does not exist."
|
||||||
|
update_status(:failed, message)
|
||||||
|
Logger.error("#{__MODULE__}: #{message}")
|
||||||
|
|
||||||
|
data_migration.state == :manual or data_migration.name in manual_migrations ->
|
||||||
|
message = "Data migration is in manual execution or manual fix mode."
|
||||||
|
update_status(:manual, message)
|
||||||
|
Logger.warn("#{__MODULE__}: #{message}")
|
||||||
|
|
||||||
|
data_migration.state == :complete ->
|
||||||
|
on_complete(data_migration)
|
||||||
|
|
||||||
|
true ->
|
||||||
|
send(self(), :perform)
|
||||||
|
end
|
||||||
|
|
||||||
|
{:noreply, nil}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info(:perform, state) do
|
||||||
|
State.reinit()
|
||||||
|
|
||||||
|
update_status(:running)
|
||||||
|
put_stat(:iteration_processed_count, 0)
|
||||||
|
put_stat(:started_at, NaiveDateTime.utc_now())
|
||||||
|
|
||||||
|
perform()
|
||||||
|
|
||||||
|
fault_rate = fault_rate()
|
||||||
|
put_stat(:fault_rate, fault_rate)
|
||||||
|
fault_rate_allowance = fault_rate_allowance()
|
||||||
|
|
||||||
|
cond do
|
||||||
|
fault_rate == 0 ->
|
||||||
|
set_complete()
|
||||||
|
|
||||||
|
is_float(fault_rate) and fault_rate <= fault_rate_allowance ->
|
||||||
|
message = """
|
||||||
|
Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}.
|
||||||
|
Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
Logger.warn("#{__MODULE__}: #{message}")
|
||||||
|
update_status(:manual, message)
|
||||||
|
on_complete(data_migration())
|
||||||
|
|
||||||
|
true ->
|
||||||
|
message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`."
|
||||||
|
Logger.error("#{__MODULE__}: #{message}")
|
||||||
|
update_status(:failed, message)
|
||||||
|
end
|
||||||
|
|
||||||
|
persist_state()
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp on_complete(data_migration) do
|
||||||
|
if data_migration.feature_lock || feature_state() == :disabled do
|
||||||
|
Logger.warn(
|
||||||
|
"#{__MODULE__}: migration complete but feature is locked; consider enabling."
|
||||||
|
)
|
||||||
|
|
||||||
|
:noop
|
||||||
|
else
|
||||||
|
Config.put(feature_config_path(), :enabled)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc "Approximate count for current iteration (including processed records count)"
|
||||||
|
def count(force \\ false, timeout \\ :infinity) do
|
||||||
|
stored_count = get_stat(:count)
|
||||||
|
|
||||||
|
if stored_count && !force do
|
||||||
|
stored_count
|
||||||
|
else
|
||||||
|
processed_count = get_stat(:processed_count, 0)
|
||||||
|
max_processed_id = get_stat(:max_processed_id, 0)
|
||||||
|
query = where(query(), [entity], entity.id > ^max_processed_id)
|
||||||
|
|
||||||
|
count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
|
||||||
|
put_stat(:count, count)
|
||||||
|
persist_state()
|
||||||
|
|
||||||
|
count
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def failures_count do
|
||||||
|
with {:ok, %{rows: [[count]]}} <-
|
||||||
|
Repo.query(
|
||||||
|
"SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
|
||||||
|
[data_migration_id()]
|
||||||
|
) do
|
||||||
|
count
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def feature_state, do: Config.get(feature_config_path())
|
||||||
|
|
||||||
|
def force_continue do
|
||||||
|
send(whereis(), :perform)
|
||||||
|
end
|
||||||
|
|
||||||
|
def force_restart do
|
||||||
|
:ok = State.reset()
|
||||||
|
force_continue()
|
||||||
|
end
|
||||||
|
|
||||||
|
def set_complete do
|
||||||
|
update_status(:complete)
|
||||||
|
persist_state()
|
||||||
|
on_complete(data_migration())
|
||||||
|
end
|
||||||
|
|
||||||
|
defp update_status(status, message \\ nil) do
|
||||||
|
put_stat(:state, status)
|
||||||
|
put_stat(:message, message)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fault_rate do
|
||||||
|
with failures_count when is_integer(failures_count) <- failures_count() do
|
||||||
|
failures_count / Enum.max([get_stat(:affected_count, 0), 1])
|
||||||
|
else
|
||||||
|
_ -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp records_per_second do
|
||||||
|
get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp running_time do
|
||||||
|
NaiveDateTime.diff(
|
||||||
|
NaiveDateTime.utc_now(),
|
||||||
|
get_stat(:started_at, NaiveDateTime.utc_now())
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,117 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Migrators.Support.BaseMigratorState do
|
||||||
|
@moduledoc """
|
||||||
|
Base background migrator state functionality.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@callback data_migration() :: Pleroma.DataMigration.t()
|
||||||
|
|
||||||
|
defmacro __using__(_opts) do
|
||||||
|
quote do
|
||||||
|
use Agent
|
||||||
|
|
||||||
|
alias Pleroma.DataMigration
|
||||||
|
|
||||||
|
@behaviour Pleroma.Migrators.Support.BaseMigratorState
|
||||||
|
@reg_name {:global, __MODULE__}
|
||||||
|
|
||||||
|
def start_link(_) do
|
||||||
|
Agent.start_link(fn -> load_state_from_db() end, name: @reg_name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def data_migration, do: raise("data_migration/0 is not implemented")
|
||||||
|
defoverridable data_migration: 0
|
||||||
|
|
||||||
|
defp load_state_from_db do
|
||||||
|
data_migration = data_migration()
|
||||||
|
|
||||||
|
data =
|
||||||
|
if data_migration do
|
||||||
|
Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end)
|
||||||
|
else
|
||||||
|
%{}
|
||||||
|
end
|
||||||
|
|
||||||
|
%{
|
||||||
|
data_migration_id: data_migration && data_migration.id,
|
||||||
|
data: data
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def persist_to_db do
|
||||||
|
%{data_migration_id: data_migration_id, data: data} = state()
|
||||||
|
|
||||||
|
if data_migration_id do
|
||||||
|
DataMigration.update_one_by_id(data_migration_id, data: data)
|
||||||
|
else
|
||||||
|
{:error, :nil_data_migration_id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def reset do
|
||||||
|
%{data_migration_id: data_migration_id} = state()
|
||||||
|
|
||||||
|
with false <- is_nil(data_migration_id),
|
||||||
|
:ok <-
|
||||||
|
DataMigration.update_one_by_id(data_migration_id,
|
||||||
|
state: :pending,
|
||||||
|
data: %{}
|
||||||
|
) do
|
||||||
|
reinit()
|
||||||
|
else
|
||||||
|
true -> {:error, :nil_data_migration_id}
|
||||||
|
e -> e
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def reinit do
|
||||||
|
Agent.update(@reg_name, fn _state -> load_state_from_db() end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def state do
|
||||||
|
Agent.get(@reg_name, & &1)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_data_key(key, default \\ nil) do
|
||||||
|
get_in(state(), [:data, key]) || default
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_data_key(key, value) do
|
||||||
|
_ = persist_non_data_change(key, value)
|
||||||
|
|
||||||
|
Agent.update(@reg_name, fn state ->
|
||||||
|
put_in(state, [:data, key], value)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def increment_data_key(key, increment \\ 1) do
|
||||||
|
Agent.update(@reg_name, fn state ->
|
||||||
|
initial_value = get_in(state, [:data, key]) || 0
|
||||||
|
updated_value = initial_value + increment
|
||||||
|
put_in(state, [:data, key], updated_value)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp persist_non_data_change(:state, value) do
|
||||||
|
with true <- get_data_key(:state) != value,
|
||||||
|
true <- value in Pleroma.DataMigration.State.__valid_values__(),
|
||||||
|
%{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <-
|
||||||
|
state() do
|
||||||
|
DataMigration.update_one_by_id(data_migration_id, state: value)
|
||||||
|
else
|
||||||
|
false -> :ok
|
||||||
|
_ -> {:error, :nil_data_migration_id}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp persist_non_data_change(_, _) do
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def data_migration_id, do: Map.get(state(), :data_migration_id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,116 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
|
||||||
|
require Pleroma.Constants
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Object
|
||||||
|
|
||||||
|
@moduledoc """
|
||||||
|
Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #)
|
||||||
|
|
||||||
|
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@behaviour Pleroma.Web.ActivityPub.MRF
|
||||||
|
|
||||||
|
defp check_reject(message, hashtags) do
|
||||||
|
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
|
||||||
|
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp check_ftl_removal(%{"to" => to} = message, hashtags) do
|
||||||
|
if Pleroma.Constants.as_public() in to and
|
||||||
|
Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
|
||||||
|
match in hashtags
|
||||||
|
end) do
|
||||||
|
to = List.delete(to, Pleroma.Constants.as_public())
|
||||||
|
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
|
||||||
|
|
||||||
|
message =
|
||||||
|
message
|
||||||
|
|> Map.put("to", to)
|
||||||
|
|> Map.put("cc", cc)
|
||||||
|
|> Kernel.put_in(["object", "to"], to)
|
||||||
|
|> Kernel.put_in(["object", "cc"], cc)
|
||||||
|
|
||||||
|
{:ok, message}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
|
||||||
|
|
||||||
|
defp check_sensitive(message, hashtags) do
|
||||||
|
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
|
||||||
|
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def filter(%{"type" => "Create", "object" => object} = message) do
|
||||||
|
hashtags = Object.hashtags(%Object{data: object})
|
||||||
|
|
||||||
|
if hashtags != [] do
|
||||||
|
with {:ok, message} <- check_reject(message, hashtags),
|
||||||
|
{:ok, message} <- check_ftl_removal(message, hashtags),
|
||||||
|
{:ok, message} <- check_sensitive(message, hashtags) do
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
else
|
||||||
|
{:ok, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def filter(message), do: {:ok, message}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def describe do
|
||||||
|
mrf_hashtag =
|
||||||
|
Config.get(:mrf_hashtag)
|
||||||
|
|> Enum.into(%{})
|
||||||
|
|
||||||
|
{:ok, %{mrf_hashtag: mrf_hashtag}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def config_description do
|
||||||
|
%{
|
||||||
|
key: :mrf_hashtag,
|
||||||
|
related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy",
|
||||||
|
label: "MRF Hashtag",
|
||||||
|
description: @moduledoc,
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :reject,
|
||||||
|
type: {:list, :string},
|
||||||
|
description: "A list of hashtags which result in message being rejected.",
|
||||||
|
suggestions: ["foo"]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :federated_timeline_removal,
|
||||||
|
type: {:list, :string},
|
||||||
|
description:
|
||||||
|
"A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
|
||||||
|
suggestions: ["foo"]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :sensitive,
|
||||||
|
type: {:list, :string},
|
||||||
|
description:
|
||||||
|
"A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
|
||||||
|
suggestions: ["nsfw", "r18"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,389 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do
|
||||||
|
alias OpenApiSpex.Operation
|
||||||
|
alias OpenApiSpex.Schema
|
||||||
|
alias Pleroma.Web.ApiSpec.Schemas.ActorType
|
||||||
|
alias Pleroma.Web.ApiSpec.Schemas.ApiError
|
||||||
|
|
||||||
|
import Pleroma.Web.ApiSpec.Helpers
|
||||||
|
|
||||||
|
def open_api_operation(action) do
|
||||||
|
operation = String.to_existing_atom("#{action}_operation")
|
||||||
|
apply(__MODULE__, operation, [])
|
||||||
|
end
|
||||||
|
|
||||||
|
def index_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "List users",
|
||||||
|
operationId: "AdminAPI.UserController.index",
|
||||||
|
security: [%{"oAuth" => ["admin:read:accounts"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(:filters, :query, :string, "Comma separated list of filters"),
|
||||||
|
Operation.parameter(:query, :query, :string, "Search users query"),
|
||||||
|
Operation.parameter(:name, :query, :string, "Search by display name"),
|
||||||
|
Operation.parameter(:email, :query, :string, "Search by email"),
|
||||||
|
Operation.parameter(:page, :query, :integer, "Page Number"),
|
||||||
|
Operation.parameter(:page_size, :query, :integer, "Number of users to return per page"),
|
||||||
|
Operation.parameter(
|
||||||
|
:actor_types,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :array, items: ActorType},
|
||||||
|
"Filter by actor type"
|
||||||
|
),
|
||||||
|
Operation.parameter(
|
||||||
|
:tags,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :array, items: %Schema{type: :string}},
|
||||||
|
"Filter by tags"
|
||||||
|
)
|
||||||
|
| admin_api_params()
|
||||||
|
],
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response(
|
||||||
|
"Response",
|
||||||
|
"application/json",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
users: %Schema{type: :array, items: user()},
|
||||||
|
count: %Schema{type: :integer},
|
||||||
|
page_size: %Schema{type: :integer}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def create_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Create a single or multiple users",
|
||||||
|
operationId: "AdminAPI.UserController.create",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
description: "POST body for creating users",
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
users: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
nickname: %Schema{type: :string},
|
||||||
|
email: %Schema{type: :string},
|
||||||
|
password: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response("Response", "application/json", %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
code: %Schema{type: :integer},
|
||||||
|
type: %Schema{type: :string},
|
||||||
|
data: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
email: %Schema{type: :string, format: :email},
|
||||||
|
nickname: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError),
|
||||||
|
409 =>
|
||||||
|
Operation.response("Conflict", "application/json", %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
code: %Schema{type: :integer},
|
||||||
|
error: %Schema{type: :string},
|
||||||
|
type: %Schema{type: :string},
|
||||||
|
data: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
email: %Schema{type: :string, format: :email},
|
||||||
|
nickname: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def show_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Show user",
|
||||||
|
operationId: "AdminAPI.UserController.show",
|
||||||
|
security: [%{"oAuth" => ["admin:read:accounts"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(
|
||||||
|
:nickname,
|
||||||
|
:path,
|
||||||
|
:string,
|
||||||
|
"User nickname or ID"
|
||||||
|
)
|
||||||
|
| admin_api_params()
|
||||||
|
],
|
||||||
|
responses: %{
|
||||||
|
200 => Operation.response("Response", "application/json", user()),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError),
|
||||||
|
404 => Operation.response("Not Found", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def follow_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Follow",
|
||||||
|
operationId: "AdminAPI.UserController.follow",
|
||||||
|
security: [%{"oAuth" => ["admin:write:follows"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
follower: %Schema{type: :string, description: "Follower nickname"},
|
||||||
|
followed: %Schema{type: :string, description: "Followed nickname"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => Operation.response("Response", "application/json", %Schema{type: :string}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def unfollow_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Unfollow",
|
||||||
|
operationId: "AdminAPI.UserController.unfollow",
|
||||||
|
security: [%{"oAuth" => ["admin:write:follows"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
follower: %Schema{type: :string, description: "Follower nickname"},
|
||||||
|
followed: %Schema{type: :string, description: "Followed nickname"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => Operation.response("Response", "application/json", %Schema{type: :string}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def approve_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Approve multiple users",
|
||||||
|
operationId: "AdminAPI.UserController.approve",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
description: "POST body for deleting multiple users",
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
nicknames: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response("Response", "application/json", %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{user: %Schema{type: :array, items: user()}}
|
||||||
|
}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def toggle_activation_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Toggle user activation",
|
||||||
|
operationId: "AdminAPI.UserController.toggle_activation",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(:nickname, :path, :string, "User nickname")
|
||||||
|
| admin_api_params()
|
||||||
|
],
|
||||||
|
responses: %{
|
||||||
|
200 => Operation.response("Response", "application/json", user()),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def activate_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Activate multiple users",
|
||||||
|
operationId: "AdminAPI.UserController.activate",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
description: "POST body for deleting multiple users",
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
nicknames: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response("Response", "application/json", %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{user: %Schema{type: :array, items: user()}}
|
||||||
|
}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def deactivate_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Deactivates multiple users",
|
||||||
|
operationId: "AdminAPI.UserController.deactivate",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: admin_api_params(),
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
description: "POST body for deleting multiple users",
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
nicknames: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response("Response", "application/json", %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{user: %Schema{type: :array, items: user()}}
|
||||||
|
}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["User administration"],
|
||||||
|
summary: "Removes a single or multiple users",
|
||||||
|
operationId: "AdminAPI.UserController.delete",
|
||||||
|
security: [%{"oAuth" => ["admin:write:accounts"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(
|
||||||
|
:nickname,
|
||||||
|
:query,
|
||||||
|
:string,
|
||||||
|
"User nickname"
|
||||||
|
)
|
||||||
|
| admin_api_params()
|
||||||
|
],
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
description: "POST body for deleting multiple users",
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
nicknames: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 =>
|
||||||
|
Operation.response("Response", "application/json", %Schema{
|
||||||
|
description: "Array of nicknames",
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{type: :string}
|
||||||
|
}),
|
||||||
|
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp user do
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
id: %Schema{type: :string},
|
||||||
|
email: %Schema{type: :string, format: :email},
|
||||||
|
avatar: %Schema{type: :string, format: :uri},
|
||||||
|
nickname: %Schema{type: :string},
|
||||||
|
display_name: %Schema{type: :string},
|
||||||
|
is_active: %Schema{type: :boolean},
|
||||||
|
local: %Schema{type: :boolean},
|
||||||
|
roles: %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
admin: %Schema{type: :boolean},
|
||||||
|
moderator: %Schema{type: :boolean}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
tags: %Schema{type: :array, items: %Schema{type: :string}},
|
||||||
|
is_confirmed: %Schema{type: :boolean},
|
||||||
|
is_approved: %Schema{type: :boolean},
|
||||||
|
url: %Schema{type: :string, format: :uri},
|
||||||
|
registration_reason: %Schema{type: :string, nullable: true},
|
||||||
|
actor_type: %Schema{type: :string}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,13 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.CreateHashtags do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def change do
|
||||||
|
create_if_not_exists table(:hashtags) do
|
||||||
|
add(:name, :citext, null: false)
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
create_if_not_exists(unique_index(:hashtags, [:name]))
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,15 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.RemoveDataFromHashtags do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def up do
|
||||||
|
alter table(:hashtags) do
|
||||||
|
remove_if_exists(:data, :map)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
alter table(:hashtags) do
|
||||||
|
add_if_not_exists(:data, :map, default: %{})
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,13 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def change do
|
||||||
|
create_if_not_exists table(:hashtags_objects, primary_key: false) do
|
||||||
|
add(:hashtag_id, references(:hashtags), null: false, primary_key: true)
|
||||||
|
add(:object_id, references(:objects), null: false, primary_key: true)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Note: PK index: "hashtags_objects_pkey" PRIMARY KEY, btree (hashtag_id, object_id)
|
||||||
|
create_if_not_exists(index(:hashtags_objects, [:object_id]))
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,17 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.CreateDataMigrations do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def change do
|
||||||
|
create_if_not_exists table(:data_migrations) do
|
||||||
|
add(:name, :string, null: false)
|
||||||
|
add(:state, :integer, default: 1)
|
||||||
|
add(:feature_lock, :boolean, default: false)
|
||||||
|
add(:params, :map, default: %{})
|
||||||
|
add(:data, :map, default: %{})
|
||||||
|
|
||||||
|
timestamps()
|
||||||
|
end
|
||||||
|
|
||||||
|
create_if_not_exists(unique_index(:data_migrations, [:name]))
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,16 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.DataMigrationCreatePopulateHashtagsTable do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def up do
|
||||||
|
dt = NaiveDateTime.utc_now()
|
||||||
|
|
||||||
|
execute(
|
||||||
|
"INSERT INTO data_migrations(name, inserted_at, updated_at) " <>
|
||||||
|
"VALUES ('populate_hashtags_table', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
execute("DELETE FROM data_migrations WHERE name = 'populate_hashtags_table';")
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,14 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.CreateDataMigrationFailedIds do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def change do
|
||||||
|
create_if_not_exists table(:data_migration_failed_ids, primary_key: false) do
|
||||||
|
add(:data_migration_id, references(:data_migrations), null: false, primary_key: true)
|
||||||
|
add(:record_id, :bigint, null: false, primary_key: true)
|
||||||
|
end
|
||||||
|
|
||||||
|
create_if_not_exists(
|
||||||
|
unique_index(:data_migration_failed_ids, [:data_migration_id, :record_id])
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,11 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.RemoveHashtagsObjectsDuplicateIndex do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
@moduledoc "Removes `hashtags_objects_hashtag_id_object_id_index` index (duplicate of PK index)."
|
||||||
|
|
||||||
|
def up do
|
||||||
|
drop_if_exists(unique_index(:hashtags_objects, [:hashtag_id, :object_id]))
|
||||||
|
end
|
||||||
|
|
||||||
|
def down, do: nil
|
||||||
|
end
|
@ -0,0 +1,15 @@
|
|||||||
|
defmodule Pleroma.Repo.Migrations.ChangeHashtagsNameToText do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def up do
|
||||||
|
alter table(:hashtags) do
|
||||||
|
modify(:name, :text)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
alter table(:hashtags) do
|
||||||
|
modify(:name, :citext)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"id": "https://fed.brid.gy/jk.nipponalba.scot",
|
||||||
|
"url": "https://fed.brid.gy/r/https://jk.nipponalba.scot",
|
||||||
|
"urls": [
|
||||||
|
{
|
||||||
|
"value": "https://jk.nipponalba.scot"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"value": "https://social.nipponalba.scot/jk"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"value": "https://px.nipponalba.scot/jk"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"@context": "https://www.w3.org/ns/activitystreams",
|
||||||
|
"type": "Person",
|
||||||
|
"name": "J K 🇯🇵🏴",
|
||||||
|
"image": [
|
||||||
|
{
|
||||||
|
"url": "https://jk.nipponalba.scot/images/profile.jpg",
|
||||||
|
"type": "Image",
|
||||||
|
"name": "profile picture"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tag": [
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Craft Beer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Single Malt Whisky"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Homebrewing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Scottish Politics"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Scottish History"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Japanese History"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Tech"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Veganism"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Tag",
|
||||||
|
"name": "Cooking"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"icon": [
|
||||||
|
{
|
||||||
|
"url": "https://jk.nipponalba.scot/images/profile.jpg",
|
||||||
|
"type": "Image",
|
||||||
|
"name": "profile picture"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"preferredUsername": "jk.nipponalba.scot",
|
||||||
|
"summary": "",
|
||||||
|
"publicKey": {
|
||||||
|
"id": "jk.nipponalba.scot",
|
||||||
|
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdarxwzxnNbJ2hneWOYHkYJowk\npyigQtxlUd0VjgSQHwxU9kWqfbrHBVADyTtcqi/4dAzQd3UnCI1TPNnn4LPZY9PW\noiWd3Zl1/EfLFxO7LU9GS7fcSLQkyj5JNhSlN3I8QPudZbybrgRDVZYooDe1D+52\n5KLGqC2ajrIVOiDRTQIDAQAB\n-----END PUBLIC KEY-----"
|
||||||
|
},
|
||||||
|
"inbox": "https://fed.brid.gy/jk.nipponalba.scot/inbox",
|
||||||
|
"outbox": "https://fed.brid.gy/jk.nipponalba.scot/outbox",
|
||||||
|
"following": "https://fed.brid.gy/jk.nipponalba.scot/following",
|
||||||
|
"followers": "https://fed.brid.gy/jk.nipponalba.scot/followers"
|
||||||
|
}
|
@ -0,0 +1,49 @@
|
|||||||
|
{
|
||||||
|
"@context": [
|
||||||
|
"https://www.w3.org/ns/activitystreams",
|
||||||
|
"https://patch.cx/schemas/litepub-0.1.jsonld",
|
||||||
|
{
|
||||||
|
"@language": "und"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"actor": "https://patch.cx/users/rin",
|
||||||
|
"attachment": [],
|
||||||
|
"attributedTo": "https://patch.cx/users/rin",
|
||||||
|
"cc": [
|
||||||
|
"https://patch.cx/users/rin/followers"
|
||||||
|
],
|
||||||
|
"content": ":joker_disapprove: <br><br>just grabbing a test fixture, nevermind me",
|
||||||
|
"context": "https://patch.cx/contexts/2c3ce4b4-18b1-4b1a-8965-3932027b5326",
|
||||||
|
"conversation": "https://patch.cx/contexts/2c3ce4b4-18b1-4b1a-8965-3932027b5326",
|
||||||
|
"id": "https://patch.cx/objects/a399c28e-c821-4820-bc3e-4afeb044c16f",
|
||||||
|
"published": "2021-03-22T16:54:46.461939Z",
|
||||||
|
"sensitive": null,
|
||||||
|
"source": ":joker_disapprove: \r\n\r\njust grabbing a test fixture, nevermind me",
|
||||||
|
"summary": ":joker_smile: ",
|
||||||
|
"tag": [
|
||||||
|
{
|
||||||
|
"icon": {
|
||||||
|
"type": "Image",
|
||||||
|
"url": "https://patch.cx/emoji/custom/joker_disapprove.png"
|
||||||
|
},
|
||||||
|
"id": "https://patch.cx/emoji/custom/joker_disapprove.png",
|
||||||
|
"name": ":joker_disapprove:",
|
||||||
|
"type": "Emoji",
|
||||||
|
"updated": "1970-01-01T00:00:00Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"icon": {
|
||||||
|
"type": "Image",
|
||||||
|
"url": "https://patch.cx/emoji/custom/joker_smile.png"
|
||||||
|
},
|
||||||
|
"id": "https://patch.cx/emoji/custom/joker_smile.png",
|
||||||
|
"name": ":joker_smile:",
|
||||||
|
"type": "Emoji",
|
||||||
|
"updated": "1970-01-01T00:00:00Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"to": [
|
||||||
|
"https://www.w3.org/ns/activitystreams#Public"
|
||||||
|
],
|
||||||
|
"type": "Note"
|
||||||
|
}
|
@ -1,4 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
|
|
||||||
<Link rel="lrdd" template="https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource={uri}" type="application/xrd+xml" />
|
|
||||||
</XRD>
|
|
@ -0,0 +1,17 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.HashtagTest do
|
||||||
|
use Pleroma.DataCase
|
||||||
|
|
||||||
|
alias Pleroma.Hashtag
|
||||||
|
|
||||||
|
describe "changeset validations" do
|
||||||
|
test "ensure non-blank :name" do
|
||||||
|
changeset = Hashtag.changeset(%Hashtag{}, %{name: ""})
|
||||||
|
|
||||||
|
assert {:name, {"can't be blank", [validation: :required]}} in changeset.errors
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,31 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicyTest do
|
||||||
|
use Oban.Testing, repo: Pleroma.Repo
|
||||||
|
use Pleroma.DataCase
|
||||||
|
|
||||||
|
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||||
|
alias Pleroma.Web.CommonAPI
|
||||||
|
|
||||||
|
import Pleroma.Factory
|
||||||
|
|
||||||
|
test "it sets the sensitive property with relevant hashtags" do
|
||||||
|
user = insert(:user)
|
||||||
|
|
||||||
|
{:ok, activity} = CommonAPI.post(user, %{status: "#nsfw hey"})
|
||||||
|
{:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
|
||||||
|
|
||||||
|
assert modified["object"]["sensitive"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it doesn't sets the sensitive property with irrelevant hashtags" do
|
||||||
|
user = insert(:user)
|
||||||
|
|
||||||
|
{:ok, activity} = CommonAPI.post(user, %{status: "#cofe hey"})
|
||||||
|
{:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
|
||||||
|
|
||||||
|
refute modified["object"]["sensitive"]
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in new issue