#1855 MediaProxy cache invalidation via Admin API See merge request pleroma/pleroma!2648stable
commit
f928267773
@ -0,0 +1,63 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
|
||||||
|
use Pleroma.Web, :controller
|
||||||
|
|
||||||
|
alias Pleroma.Plugs.OAuthScopesPlug
|
||||||
|
alias Pleroma.Web.ApiSpec.Admin, as: Spec
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
|
plug(Pleroma.Web.ApiSpec.CastAndValidate)
|
||||||
|
|
||||||
|
plug(
|
||||||
|
OAuthScopesPlug,
|
||||||
|
%{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
|
||||||
|
)
|
||||||
|
|
||||||
|
plug(
|
||||||
|
OAuthScopesPlug,
|
||||||
|
%{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
|
||||||
|
)
|
||||||
|
|
||||||
|
action_fallback(Pleroma.Web.AdminAPI.FallbackController)
|
||||||
|
|
||||||
|
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
|
||||||
|
|
||||||
|
def index(%{assigns: %{user: _}} = conn, params) do
|
||||||
|
cursor =
|
||||||
|
:banned_urls_cache
|
||||||
|
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|
||||||
|
|> :qlc.cursor()
|
||||||
|
|
||||||
|
urls =
|
||||||
|
case params.page do
|
||||||
|
1 ->
|
||||||
|
:qlc.next_answers(cursor, params.page_size)
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
|
||||||
|
:qlc.next_answers(cursor, params.page_size)
|
||||||
|
end
|
||||||
|
|
||||||
|
:qlc.delete_cursor(cursor)
|
||||||
|
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
|
||||||
|
MediaProxy.remove_from_banned_urls(urls)
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
|
||||||
|
MediaProxy.Invalidation.purge(urls)
|
||||||
|
|
||||||
|
if ban do
|
||||||
|
MediaProxy.put_in_banned_urls(urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,11 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
|
||||||
|
use Pleroma.Web, :view
|
||||||
|
|
||||||
|
def render("index.json", %{urls: urls}) do
|
||||||
|
%{urls: urls}
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,109 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
|
alias OpenApiSpex.Operation
|
||||||
|
alias OpenApiSpex.Schema
|
||||||
|
alias Pleroma.Web.ApiSpec.Schemas.ApiError
|
||||||
|
|
||||||
|
import Pleroma.Web.ApiSpec.Helpers
|
||||||
|
|
||||||
|
def open_api_operation(action) do
|
||||||
|
operation = String.to_existing_atom("#{action}_operation")
|
||||||
|
apply(__MODULE__, operation, [])
|
||||||
|
end
|
||||||
|
|
||||||
|
def index_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.index",
|
||||||
|
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(
|
||||||
|
:page,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :integer, default: 1},
|
||||||
|
"Page"
|
||||||
|
),
|
||||||
|
Operation.parameter(
|
||||||
|
:page_size,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :integer, default: 50},
|
||||||
|
"Number of statuses to return"
|
||||||
|
)
|
||||||
|
],
|
||||||
|
responses: %{
|
||||||
|
200 => success_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Remove a banned MediaProxy URL from Cachex",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.delete",
|
||||||
|
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
required: [:urls],
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
required: true
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => success_response(),
|
||||||
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def purge_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Purge and optionally ban a MediaProxy URL",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.purge",
|
||||||
|
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
required: [:urls],
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
|
||||||
|
ban: %Schema{type: :boolean, default: true}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
required: true
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => success_response(),
|
||||||
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp success_response do
|
||||||
|
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :string,
|
||||||
|
format: :uri,
|
||||||
|
description: "MediaProxy URLs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,145 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
|
use Pleroma.Web.ConnCase
|
||||||
|
|
||||||
|
import Pleroma.Factory
|
||||||
|
import Mock
|
||||||
|
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
|
setup do: clear_config([:media_proxy])
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
setup do
|
||||||
|
admin = insert(:user, is_admin: true)
|
||||||
|
token = insert(:oauth_admin_token, user: admin)
|
||||||
|
|
||||||
|
conn =
|
||||||
|
build_conn()
|
||||||
|
|> assign(:user, admin)
|
||||||
|
|> assign(:token, token)
|
||||||
|
|
||||||
|
Config.put([:media_proxy, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
|
||||||
|
|
||||||
|
{:ok, %{admin: admin, token: token, conn: conn}}
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "GET /api/pleroma/admin/media_proxy_caches" do
|
||||||
|
test "shows banned MediaProxy URLs", %{conn: conn} do
|
||||||
|
MediaProxy.put_in_banned_urls([
|
||||||
|
"http://localhost:4001/media/a688346.jpg",
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg"
|
||||||
|
])
|
||||||
|
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == [
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg",
|
||||||
|
"http://localhost:4001/media/a688346.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == [
|
||||||
|
"http://localhost:4001/media/gb1f44.jpg",
|
||||||
|
"http://localhost:4001/media/tb13f47.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
|
||||||
|
test "deleted MediaProxy URLs from banned", %{conn: conn} do
|
||||||
|
MediaProxy.put_in_banned_urls([
|
||||||
|
"http://localhost:4001/media/a688346.jpg",
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg"
|
||||||
|
])
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
|
||||||
|
urls: ["http://localhost:4001/media/a688346.jpg"]
|
||||||
|
})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
|
||||||
|
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
|
||||||
|
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
|
||||||
|
test "perform invalidates cache of MediaProxy", %{conn: conn} do
|
||||||
|
urls = [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
with_mocks [
|
||||||
|
{MediaProxy.Invalidation.Script, [],
|
||||||
|
[
|
||||||
|
purge: fn _, _ -> {"ok", 0} end
|
||||||
|
]}
|
||||||
|
] do
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == urls
|
||||||
|
|
||||||
|
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
|
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
|
||||||
|
urls = [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{
|
||||||
|
urls: urls,
|
||||||
|
ban: true
|
||||||
|
})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == urls
|
||||||
|
|
||||||
|
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
|
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
@ -0,0 +1,64 @@
|
|||||||
|
defmodule Pleroma.Web.MediaProxy.InvalidationTest do
|
||||||
|
use ExUnit.Case
|
||||||
|
use Pleroma.Tests.Helpers
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Web.MediaProxy.Invalidation
|
||||||
|
|
||||||
|
import ExUnit.CaptureLog
|
||||||
|
import Mock
|
||||||
|
import Tesla.Mock
|
||||||
|
|
||||||
|
setup do: clear_config([:media_proxy])
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "Invalidation.Http" do
|
||||||
|
test "perform request to clear cache" do
|
||||||
|
Config.put([:media_proxy, :enabled], false)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
|
||||||
|
|
||||||
|
Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
|
||||||
|
image_url = "http://example.com/media/example.jpg"
|
||||||
|
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
|
||||||
|
|
||||||
|
mock(fn
|
||||||
|
%{
|
||||||
|
method: :purge,
|
||||||
|
url: "http://example.com/media/example.jpg",
|
||||||
|
headers: [{"x-refresh", 1}]
|
||||||
|
} ->
|
||||||
|
%Tesla.Env{status: 200}
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
end) =~ "Running cache purge: [\"#{image_url}\"]"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "Invalidation.Script" do
|
||||||
|
test "run script to clear cache" do
|
||||||
|
Config.put([:media_proxy, :enabled], false)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
|
||||||
|
Config.put([Invalidation.Script], script_path: "purge-nginx")
|
||||||
|
|
||||||
|
image_url = "http://example.com/media/example.jpg"
|
||||||
|
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
|
||||||
|
|
||||||
|
with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
end) =~ "Running cache purge: [\"#{image_url}\"]"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in new issue