commit
489b12cde4
@ -1,2 +1,8 @@
|
||||
*.ex diff=elixir
|
||||
*.exs diff=elixir
|
||||
# At the time of writing all js/css files included
|
||||
# in the repo are minified bundles, and we don't want
|
||||
# to search/diff those as text files.
|
||||
*.js binary
|
||||
*.js.map binary
|
||||
*.css binary
|
||||
|
@ -1,9 +0,0 @@
|
||||
# Generate release environment file
|
||||
|
||||
```sh tab="OTP"
|
||||
./bin/pleroma_ctl release_env gen
|
||||
```
|
||||
|
||||
```sh tab="From Source"
|
||||
mix pleroma.release_env gen
|
||||
```
|
@ -0,0 +1,136 @@
|
||||
# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication
|
||||
|
||||
If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
|
||||
|
||||
In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully.
|
||||
|
||||
Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary.
|
||||
|
||||
```bash
|
||||
cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py
|
||||
chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py
|
||||
chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py
|
||||
```
|
||||
|
||||
Set external auth params in ejabberd.yaml file:
|
||||
|
||||
```bash
|
||||
auth_method: [external]
|
||||
extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py"
|
||||
extauth_instances: 3
|
||||
auth_use_cache: false
|
||||
```
|
||||
|
||||
Restart / reload your ejabberd service.
|
||||
|
||||
After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.
|
||||
|
||||
|
||||
```python
|
||||
import sys
|
||||
import struct
|
||||
import http.client
|
||||
from base64 import b64encode
|
||||
import logging
|
||||
|
||||
|
||||
PLEROMA_HOST = "127.0.0.1"
|
||||
PLEROMA_PORT = "4000"
|
||||
AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials"
|
||||
USER_ENDPOINT = "/api/v1/accounts"
|
||||
LOGFILE = "/var/log/ejabberd/pleroma_auth.log"
|
||||
|
||||
logging.basicConfig(filename=LOGFILE, level=logging.INFO)
|
||||
|
||||
|
||||
# Pleroma functions
|
||||
def create_connection():
|
||||
return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT)
|
||||
|
||||
|
||||
def verify_credentials(user: str, password: str) -> bool:
|
||||
user_pass_b64 = b64encode("{}:{}".format(
|
||||
user, password).encode('utf-8')).decode("ascii")
|
||||
params = {}
|
||||
headers = {
|
||||
"Authorization": "Basic {}".format(user_pass_b64)
|
||||
}
|
||||
|
||||
try:
|
||||
conn = create_connection()
|
||||
conn.request("GET", AUTH_ENDPOINT, params, headers)
|
||||
|
||||
response = conn.getresponse()
|
||||
if response.status == 200:
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.info("Can not connect: %s", str(e))
|
||||
return False
|
||||
|
||||
|
||||
def does_user_exist(user: str) -> bool:
|
||||
conn = create_connection()
|
||||
conn.request("GET", "{}/{}".format(USER_ENDPOINT, user))
|
||||
|
||||
response = conn.getresponse()
|
||||
if response.status == 200:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def auth(username: str, server: str, password: str) -> bool:
|
||||
return verify_credentials(username, password)
|
||||
|
||||
|
||||
def isuser(username, server):
|
||||
return does_user_exist(username)
|
||||
|
||||
|
||||
def read():
|
||||
(pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8'))
|
||||
pkt = sys.stdin.read(pkt_size)
|
||||
cmd = pkt.split(':')[0]
|
||||
if cmd == 'auth':
|
||||
username, server, password = pkt.split(':', 3)[1:]
|
||||
write(auth(username, server, password))
|
||||
elif cmd == 'isuser':
|
||||
username, server = pkt.split(':', 2)[1:]
|
||||
write(isuser(username, server))
|
||||
elif cmd == 'setpass':
|
||||
# u, s, p = pkt.split(':', 3)[1:]
|
||||
write(False)
|
||||
elif cmd == 'tryregister':
|
||||
# u, s, p = pkt.split(':', 3)[1:]
|
||||
write(False)
|
||||
elif cmd == 'removeuser':
|
||||
# u, s = pkt.split(':', 2)[1:]
|
||||
write(False)
|
||||
elif cmd == 'removeuser3':
|
||||
# u, s, p = pkt.split(':', 3)[1:]
|
||||
write(False)
|
||||
else:
|
||||
write(False)
|
||||
|
||||
|
||||
def write(result):
|
||||
if result:
|
||||
sys.stdout.write('\x00\x02\x00\x01')
|
||||
else:
|
||||
sys.stdout.write('\x00\x02\x00\x00')
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.info("Starting pleroma ejabberd auth daemon...")
|
||||
while True:
|
||||
try:
|
||||
read()
|
||||
except Exception as e:
|
||||
logging.info(
|
||||
"Error while processing data from ejabberd %s", str(e))
|
||||
pass
|
||||
|
||||
```
|
@ -0,0 +1,66 @@
|
||||
# Optimizing the BEAM
|
||||
|
||||
Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application so it is immediately ready to execute another task. This can be observed with utilities like **top(1)** which will show consistently high CPU usage for the process. Switching between procesess is a rather expensive operation and also clears CPU caches further affecting latency and performance. The goal of busy waiting is to avoid this penalty.
|
||||
|
||||
This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
|
||||
|
||||
More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
|
||||
|
||||
Please only change these settings if you are experiencing issues or really know what you are doing. In general, there's no need to change these settings.
|
||||
|
||||
## VPS Provider Recommendations
|
||||
|
||||
### Good
|
||||
|
||||
* Hetzner Cloud
|
||||
|
||||
### Bad
|
||||
|
||||
* AWS (known to use burst scheduling)
|
||||
|
||||
|
||||
## Example configurations
|
||||
|
||||
Tuning the BEAM requires you provide a config file normally called [vm.args](http://erlang.org/doc/man/erl.html#emulator-flags). If you are using systemd to manage the service you can modify the unit file as such:
|
||||
|
||||
`ExecStart=/usr/bin/elixir --erl '-args_file /opt/pleroma/config/vm.args' -S /usr/bin/mix phx.server`
|
||||
|
||||
Check your OS documentation to adopt a similar strategy on other platforms.
|
||||
|
||||
### Virtual Machine and/or few CPU cores
|
||||
|
||||
Disable the busy-waiting. This should generally only be done if you're on a platform that does burst scheduling, like AWS.
|
||||
|
||||
**vm.args:**
|
||||
|
||||
```
|
||||
+sbwt none
|
||||
+sbwtdcpu none
|
||||
+sbwtdio none
|
||||
```
|
||||
|
||||
### Dedicated Hardware
|
||||
|
||||
Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports. You can use this if you run on dedicated hardware, but it is not necessary.
|
||||
|
||||
**vm.args:**
|
||||
|
||||
```
|
||||
+P 16777216
|
||||
+Q 16777216
|
||||
+K true
|
||||
+A 128
|
||||
+sbt db
|
||||
+sbwt very_long
|
||||
+swt very_low
|
||||
+sub true
|
||||
+Mulmbcs 32767
|
||||
+Mumbcgs 1
|
||||
+Musmbcs 2047
|
||||
```
|
||||
|
||||
## Additional Reading
|
||||
|
||||
* [WhatsApp: Scaling to Millions of Simultaneous Connections](https://www.erlang-factory.com/upload/presentations/558/efsf2012-whatsapp-scaling.pdf)
|
||||
* [Preemptive Scheduling and Spinlocks](https://www.uio.no/studier/emner/matnat/ifi/nedlagte-emner/INF3150/h03/annet/slides/preemptive.pdf)
|
||||
* [The Curious Case of BEAM CPU Usage](https://stressgrid.com/blog/beam_cpu_usage/)
|
@ -1,76 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Mix.Tasks.Pleroma.ReleaseEnv do
|
||||
use Mix.Task
|
||||
import Mix.Pleroma
|
||||
|
||||
@shortdoc "Generate Pleroma environment file."
|
||||
@moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
|
||||
|
||||
def run(["gen" | rest]) do
|
||||
{options, [], []} =
|
||||
OptionParser.parse(
|
||||
rest,
|
||||
strict: [
|
||||
force: :boolean,
|
||||
path: :string
|
||||
],
|
||||
aliases: [
|
||||
p: :path,
|
||||
f: :force
|
||||
]
|
||||
)
|
||||
|
||||
file_path =
|
||||
get_option(
|
||||
options,
|
||||
:path,
|
||||
"Environment file path",
|
||||
"./config/pleroma.env"
|
||||
)
|
||||
|
||||
env_path = Path.expand(file_path)
|
||||
|
||||
proceed? =
|
||||
if File.exists?(env_path) do
|
||||
get_option(
|
||||
options,
|
||||
:force,
|
||||
"Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
|
||||
"n"
|
||||
) === "y"
|
||||
else
|
||||
true
|
||||
end
|
||||
|
||||
if proceed? do
|
||||
case do_generate(env_path) do
|
||||
{:error, reason} ->
|
||||
shell_error(
|
||||
File.Error.message(%{action: "write to file", reason: reason, path: env_path})
|
||||
)
|
||||
|
||||
_ ->
|
||||
shell_info("\nThe file generated: #{env_path}.\n")
|
||||
|
||||
shell_info("""
|
||||
WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
|
||||
Example:
|
||||
chmod 0444 #{file_path}
|
||||
chattr +i #{file_path}
|
||||
""")
|
||||
end
|
||||
else
|
||||
shell_info("\nThe file is exist. #{env_path}.\n")
|
||||
end
|
||||
end
|
||||
|
||||
def do_generate(path) do
|
||||
content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
|
||||
|
||||
File.mkdir_p!(Path.dirname(path))
|
||||
File.write(path, content)
|
||||
end
|
||||
end
|
@ -0,0 +1,110 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Frontend do
|
||||
alias Pleroma.Config
|
||||
|
||||
require Logger
|
||||
|
||||
def install(name, opts \\ []) do
|
||||
frontend_info = %{
|
||||
"ref" => opts[:ref],
|
||||
"build_url" => opts[:build_url],
|
||||
"build_dir" => opts[:build_dir]
|
||||
}
|
||||
|
||||
frontend_info =
|
||||
[:frontends, :available, name]
|
||||
|> Config.get(%{})
|
||||
|> Map.merge(frontend_info, fn _key, config, cmd ->
|
||||
# This only overrides things that are actually set
|
||||
cmd || config
|
||||
end)
|
||||
|
||||
ref = frontend_info["ref"]
|
||||
|
||||
unless ref do
|
||||
raise "No ref given or configured"
|
||||
end
|
||||
|
||||
dest = Path.join([dir(), name, ref])
|
||||
|
||||
label = "#{name} (#{ref})"
|
||||
tmp_dir = Path.join(dir(), "tmp")
|
||||
|
||||
with {_, :ok} <-
|
||||
{:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, opts[:file])},
|
||||
Logger.info("Installing #{label} to #{dest}"),
|
||||
:ok <- install_frontend(frontend_info, tmp_dir, dest) do
|
||||
File.rm_rf!(tmp_dir)
|
||||
Logger.info("Frontend #{label} installed to #{dest}")
|
||||
else
|
||||
{:download_or_unzip, _} ->
|
||||
Logger.info("Could not download or unzip the frontend")
|
||||
{:error, "Could not download or unzip the frontend"}
|
||||
|
||||
_e ->
|
||||
Logger.info("Could not install the frontend")
|
||||
{:error, "Could not install the frontend"}
|
||||
end
|
||||
end
|
||||
|
||||
def dir(opts \\ []) do
|
||||
if is_nil(opts[:static_dir]) do
|
||||
Pleroma.Config.get!([:instance, :static_dir])
|
||||
else
|
||||
opts[:static_dir]
|
||||
end
|
||||
|> Path.join("frontends")
|
||||
end
|
||||
|
||||
defp download_or_unzip(frontend_info, temp_dir, nil),
|
||||
do: download_build(frontend_info, temp_dir)
|
||||
|
||||
defp download_or_unzip(_frontend_info, temp_dir, file) do
|
||||
with {:ok, zip} <- File.read(Path.expand(file)) do
|
||||
unzip(zip, temp_dir)
|
||||
end
|
||||
end
|
||||
|
||||
def unzip(zip, dest) do
|
||||
with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
|
||||
File.rm_rf!(dest)
|
||||
File.mkdir_p!(dest)
|
||||
|
||||
Enum.each(unzipped, fn {filename, data} ->
|
||||
path = filename
|
||||
|
||||
new_file_path = Path.join(dest, path)
|
||||
|
||||
new_file_path
|
||||
|> Path.dirname()
|
||||
|> File.mkdir_p!()
|
||||
|
||||
File.write!(new_file_path, data)
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
defp download_build(frontend_info, dest) do
|
||||
Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
|
||||
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
|
||||
|
||||
with {:ok, %{status: 200, body: zip_body}} <-
|
||||
Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
|
||||
unzip(zip_body, dest)
|
||||
else
|
||||
{:error, e} -> {:error, e}
|
||||
e -> {:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
defp install_frontend(frontend_info, source, dest) do
|
||||
from = frontend_info["build_dir"] || "dist"
|
||||
File.rm_rf!(dest)
|
||||
File.mkdir_p!(dest)
|
||||
File.cp_r!(Path.join([source, from]), dest)
|
||||
:ok
|
||||
end
|
||||
end
|
@ -0,0 +1,40 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.AdminAPI.FrontendController do
|
||||
use Pleroma.Web, :controller
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Web.Plugs.OAuthScopesPlug
|
||||
|
||||
plug(Pleroma.Web.ApiSpec.CastAndValidate)
|
||||
plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :install)
|
||||
plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :index)
|
||||
action_fallback(Pleroma.Web.AdminAPI.FallbackController)
|
||||
|
||||
defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.FrontendOperation
|
||||
|
||||
def index(conn, _params) do
|
||||
installed = installed()
|
||||
|
||||
frontends =
|
||||
[:frontends, :available]
|
||||
|> Config.get([])
|
||||
|> Enum.map(fn {name, desc} ->
|
||||
Map.put(desc, "installed", name in installed)
|
||||
end)
|
||||
|
||||
render(conn, "index.json", frontends: frontends)
|
||||
end
|
||||
|
||||
def install(%{body_params: params} = conn, _params) do
|
||||
with :ok <- Pleroma.Frontend.install(params.name, Map.delete(params, :name)) do
|
||||
index(conn, %{})
|
||||
end
|
||||
end
|
||||
|
||||
defp installed do
|
||||
File.ls!(Pleroma.Frontend.dir())
|
||||
end
|
||||
end
|
@ -0,0 +1,21 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.AdminAPI.FrontendView do
|
||||
use Pleroma.Web, :view
|
||||
|
||||
def render("index.json", %{frontends: frontends}) do
|
||||
render_many(frontends, __MODULE__, "show.json")
|
||||
end
|
||||
|
||||
def render("show.json", %{frontend: frontend}) do
|
||||
%{
|
||||
name: frontend["name"],
|
||||
git: frontend["git"],
|
||||
build_url: frontend["build_url"],
|
||||
ref: frontend["ref"],
|
||||
installed: frontend["installed"]
|
||||
}
|
||||
end
|
||||
end
|
@ -0,0 +1,85 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do
|
||||
alias OpenApiSpex.Operation
|
||||
alias OpenApiSpex.Schema
|
||||
alias Pleroma.Web.ApiSpec.Schemas.ApiError
|
||||
|
||||
import Pleroma.Web.ApiSpec.Helpers
|
||||
|
||||
def open_api_operation(action) do
|
||||
operation = String.to_existing_atom("#{action}_operation")
|
||||
apply(__MODULE__, operation, [])
|
||||
end
|
||||
|
||||
def index_operation do
|
||||
%Operation{
|
||||
tags: ["Admin", "Reports"],
|
||||
summary: "Get a list of available frontends",
|
||||
operationId: "AdminAPI.FrontendController.index",
|
||||
security: [%{"oAuth" => ["read"]}],
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", list_of_frontends()),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def install_operation do
|
||||
%Operation{
|
||||
tags: ["Admin", "Reports"],
|
||||
summary: "Install a frontend",
|
||||
operationId: "AdminAPI.FrontendController.install",
|
||||
security: [%{"oAuth" => ["read"]}],
|
||||
requestBody: request_body("Parameters", install_request(), required: true),
|
||||
responses: %{
|
||||
200 => Operation.response("Response", "application/json", list_of_frontends()),
|
||||
403 => Operation.response("Forbidden", "application/json", ApiError),
|
||||
400 => Operation.response("Error", "application/json", ApiError)
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp list_of_frontends do
|
||||
%Schema{
|
||||
type: :array,
|
||||
items: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
name: %Schema{type: :string},
|
||||
git: %Schema{type: :string, format: :uri, nullable: true},
|
||||
build_url: %Schema{type: :string, format: :uri, nullable: true},
|
||||
ref: %Schema{type: :string},
|
||||
installed: %Schema{type: :boolean}
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp install_request do
|
||||
%Schema{
|
||||
title: "FrontendInstallRequest",
|
||||
type: :object,
|
||||
required: [:name],
|
||||
properties: %{
|
||||
name: %Schema{
|
||||
type: :string
|
||||
},
|
||||
ref: %Schema{
|
||||
type: :string
|
||||
},
|
||||
file: %Schema{
|
||||
type: :string
|
||||
},
|
||||
build_url: %Schema{
|
||||
type: :string
|
||||
},
|
||||
build_dir: %Schema{
|
||||
type: :string
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
end
|
@ -0,0 +1,40 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ApiSpec.PleromaInstancesOperation do
|
||||
alias OpenApiSpex.Operation
|
||||
alias OpenApiSpex.Schema
|
||||
|
||||
def open_api_operation(action) do
|
||||
operation = String.to_existing_atom("#{action}_operation")
|
||||
apply(__MODULE__, operation, [])
|
||||
end
|
||||
|
||||
def show_operation do
|
||||
%Operation{
|
||||
tags: ["PleromaInstances"],
|
||||
summary: "Instances federation status",
|
||||
description: "Information about instances deemed unreachable by the server",
|
||||
operationId: "PleromaInstances.show",
|
||||
responses: %{
|
||||
200 => Operation.response("PleromaInstances", "application/json", pleroma_instances())
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def pleroma_instances do
|
||||
%Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
unreachable: %Schema{
|
||||
type: :object,
|
||||
properties: %{hostname: %Schema{type: :string, format: :"date-time"}}
|
||||
}
|
||||
},
|
||||
example: %{
|
||||
"unreachable" => %{"consistently-unreachable.name" => "2020-10-14 22:07:58.216473"}
|
||||
}
|
||||
}
|
||||
end
|
||||
end
|
@ -1,185 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets do
|
||||
@moduledoc """
|
||||
This documents the FedSockets framework. A framework for federating
|
||||
ActivityPub objects between servers via persistant WebSocket connections.
|
||||
|
||||
FedSockets allow servers to authenticate on first contact and maintain that
|
||||
connection, eliminating the need to authenticate every time data needs to be shared.
|
||||
|
||||
## Protocol
|
||||
FedSockets currently support 2 types of data transfer:
|
||||
* `publish` method which doesn't require a response
|
||||
* `fetch` method requires a response be sent
|
||||
|
||||
### Publish
|
||||
The publish operation sends a json encoded map of the shape:
|
||||
%{action: :publish, data: json}
|
||||
and accepts (but does not require) a reply of form:
|
||||
%{"action" => "publish_reply"}
|
||||
|
||||
The outgoing params represent
|
||||
* data: ActivityPub object encoded into json
|
||||
|
||||
|
||||
### Fetch
|
||||
The fetch operation sends a json encoded map of the shape:
|
||||
%{action: :fetch, data: id, uuid: fetch_uuid}
|
||||
and requires a reply of form:
|
||||
%{"action" => "fetch_reply", "uuid" => uuid, "data" => data}
|
||||
|
||||
The outgoing params represent
|
||||
* id: an ActivityPub object URI
|
||||
* uuid: a unique uuid generated by the sender
|
||||
|
||||
The reply params represent
|
||||
* data: an ActivityPub object encoded into json
|
||||
* uuid: the uuid sent along with the fetch request
|
||||
|
||||
## Examples
|
||||
Clients of FedSocket transfers shouldn't need to use any of the functions outside of this module.
|
||||
|
||||
A typical publish operation can be performed through the following code, and a fetch operation in a similar manner.
|
||||
|
||||
case FedSockets.get_or_create_fed_socket(inbox) do
|
||||
{:ok, fedsocket} ->
|
||||
FedSockets.publish(fedsocket, json)
|
||||
|
||||
_ ->
|
||||
alternative_publish(inbox, actor, json, params)
|
||||
end
|
||||
|
||||
## Configuration
|
||||
FedSockets have the following config settings
|
||||
|
||||
config :pleroma, :fed_sockets,
|
||||
enabled: true,
|
||||
ping_interval: :timer.seconds(15),
|
||||
connection_duration: :timer.hours(1),
|
||||
rejection_duration: :timer.hours(1),
|
||||
fed_socket_fetches: [
|
||||
default: 12_000,
|
||||
interval: 3_000,
|
||||
lazy: false
|
||||
]
|
||||
* enabled - turn FedSockets on or off with this flag. Can be toggled at runtime.
|
||||
* connection_duration - How long a FedSocket can sit idle before it's culled.
|
||||
* rejection_duration - After failing to make a FedSocket connection a host will be excluded
|
||||
from further connections for this amount of time
|
||||
* fed_socket_fetches - Use these parameters to pass options to the Cachex queue backing the FetchRegistry
|
||||
* fed_socket_rejections - Use these parameters to pass options to the Cachex queue backing the FedRegistry
|
||||
|
||||
Cachex options are
|
||||
* default: the minimum amount of time a fetch can wait before it times out.
|
||||
* interval: the interval between checks for timed out entries. This plus the default represent the maximum time allowed
|
||||
* lazy: leave at false for consistant and fast lookups, set to true for stricter timeout enforcement
|
||||
|
||||
"""
|
||||
require Logger
|
||||
|
||||
alias Pleroma.Web.FedSockets.FedRegistry
|
||||
alias Pleroma.Web.FedSockets.FedSocket
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
|
||||
@doc """
|
||||
returns a FedSocket for the given origin. Will reuse an existing one or create a new one.
|
||||
|
||||
address is expected to be a fully formed URL such as:
|
||||
"http://www.example.com" or "http://www.example.com:8080"
|
||||
|
||||
It can and usually does include additional path parameters,
|
||||
but these are ignored as the FedSockets are organized by host and port info alone.
|
||||
"""
|
||||
def get_or_create_fed_socket(address) do
|
||||
with {:cache, {:error, :missing}} <- {:cache, get_fed_socket(address)},
|
||||
{:connect, {:ok, _pid}} <- {:connect, FedSocket.connect_to_host(address)},
|
||||
{:cache, {:ok, fed_socket}} <- {:cache, get_fed_socket(address)} do
|
||||
Logger.debug("fedsocket created for - #{inspect(address)}")
|
||||
{:ok, fed_socket}
|
||||
else
|
||||
{:cache, {:ok, socket}} ->
|
||||
Logger.debug("fedsocket found in cache - #{inspect(address)}")
|
||||
{:ok, socket}
|
||||
|
||||
{:cache, {:error, :rejected} = e} ->
|
||||
e
|
||||
|
||||
{:connect, {:error, _host}} ->
|
||||
Logger.debug("set host rejected for - #{inspect(address)}")
|
||||
FedRegistry.set_host_rejected(address)
|
||||
{:error, :rejected}
|
||||
|
||||
{_, {:error, :disabled}} ->
|
||||
{:error, :disabled}
|
||||
|
||||
{_, {:error, reason}} ->
|
||||
Logger.warn("get_or_create_fed_socket error - #{inspect(reason)}")
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
returns a FedSocket for the given origin. Will not create a new FedSocket if one does not exist.
|
||||
|
||||
address is expected to be a fully formed URL such as:
|
||||
"http://www.example.com" or "http://www.example.com:8080"
|
||||
"""
|
||||
def get_fed_socket(address) do
|
||||
origin = SocketInfo.origin(address)
|
||||
|
||||
with {:config, true} <- {:config, Pleroma.Config.get([:fed_sockets, :enabled], false)},
|
||||
{:ok, socket} <- FedRegistry.get_fed_socket(origin) do
|
||||
{:ok, socket}
|
||||
else
|
||||
{:config, _} ->
|
||||
{:error, :disabled}
|
||||
|
||||
{:error, :rejected} ->
|
||||
Logger.debug("FedSocket previously rejected - #{inspect(origin)}")
|
||||
{:error, :rejected}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Sends the supplied data via the publish protocol.
|
||||
It will not block waiting for a reply.
|
||||
Returns :ok but this is not an indication of a successful transfer.
|
||||
|
||||
the data is expected to be JSON encoded binary data.
|
||||
"""
|
||||
def publish(%SocketInfo{} = fed_socket, json) do
|
||||
FedSocket.publish(fed_socket, json)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Sends the supplied data via the fetch protocol.
|
||||
It will block waiting for a reply or timeout.
|
||||
|
||||
Returns {:ok, object} where object is the requested object (or nil)
|
||||
{:error, :timeout} in the event the message was not responded to
|
||||
|
||||
the id is expected to be the URI of an ActivityPub object.
|
||||
"""
|
||||
def fetch(%SocketInfo{} = fed_socket, id) do
|
||||
FedSocket.fetch(fed_socket, id)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Disconnect all and restart FedSockets.
|
||||
This is mainly used in development and testing but could be useful in production.
|
||||
"""
|
||||
def reset do
|
||||
FedRegistry
|
||||
|> Process.whereis()
|
||||
|> Process.exit(:testing)
|
||||
end
|
||||
|
||||
def uri_for_origin(origin),
|
||||
do: "ws://#{origin}/api/fedsocket/v1"
|
||||
end
|
@ -1,185 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.FedRegistry do
|
||||
@moduledoc """
|
||||
The FedRegistry stores the active FedSockets for quick retrieval.
|
||||
|
||||
The storage and retrieval portion of the FedRegistry is done in process through
|
||||
elixir's `Registry` module for speed and its ability to monitor for terminated processes.
|
||||
|
||||
Dropped connections will be caught by `Registry` and deleted. Since the next
|
||||
message will initiate a new connection there is no reason to try and reconnect at that point.
|
||||
|
||||
Normally outside modules should have no need to call or use the FedRegistry themselves.
|
||||
"""
|
||||
|
||||
alias Pleroma.Web.FedSockets.FedSocket
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
|
||||
require Logger
|
||||
|
||||
@default_rejection_duration 15 * 60 * 1000
|
||||
@rejections :fed_socket_rejections
|
||||
|
||||
@doc """
|
||||
Retrieves a FedSocket from the Registry given it's origin.
|
||||
|
||||
The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
|
||||
|
||||
Will return:
|
||||
* {:ok, fed_socket} for working FedSockets
|
||||
* {:error, :rejected} for origins that have been tried and refused within the rejection duration interval
|
||||
* {:error, some_reason} usually :missing for unknown origins
|
||||
"""
|
||||
def get_fed_socket(origin) do
|
||||
case get_registry_data(origin) do
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
{:ok, %{state: :connected} = socket_info} ->
|
||||
{:ok, socket_info}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Adds a connected FedSocket to the Registry.
|
||||
|
||||
Always returns {:ok, fed_socket}
|
||||
"""
|
||||
def add_fed_socket(origin, pid \\ nil) do
|
||||
origin
|
||||
|> SocketInfo.build(pid)
|
||||
|> SocketInfo.connect()
|
||||
|> add_socket_info
|
||||
end
|
||||
|
||||
defp add_socket_info(%{origin: origin, state: :connected} = socket_info) do
|
||||
case Registry.register(FedSockets.Registry, origin, socket_info) do
|
||||
{:ok, _owner} ->
|
||||
clear_prior_rejection(origin)
|
||||
Logger.debug("fedsocket added: #{inspect(origin)}")
|
||||
|
||||
{:ok, socket_info}
|
||||
|
||||
{:error, {:already_registered, _pid}} ->
|
||||
FedSocket.close(socket_info)
|
||||
existing_socket_info = Registry.lookup(FedSockets.Registry, origin)
|
||||
|
||||
{:ok, existing_socket_info}
|
||||
|
||||
_ ->
|
||||
{:error, :error_adding_socket}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Mark this origin as having rejected a connection attempt.
|
||||
This will keep it from getting additional connection attempts
|
||||
for a period of time specified in the config.
|
||||
|
||||
Always returns {:ok, new_reg_data}
|
||||
"""
|
||||
def set_host_rejected(uri) do
|
||||
new_reg_data =
|
||||
uri
|
||||
|> SocketInfo.origin()
|
||||
|> get_or_create_registry_data()
|
||||
|> set_to_rejected()
|
||||
|> save_registry_data()
|
||||
|
||||
{:ok, new_reg_data}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Retrieves the FedRegistryData from the Registry given it's origin.
|
||||
|
||||
The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
|
||||
|
||||
Will return:
|
||||
* {:ok, fed_registry_data} for known origins
|
||||
* {:error, :missing} for uniknown origins
|
||||
* {:error, :cache_error} indicating some low level runtime issues
|
||||
"""
|
||||
def get_registry_data(origin) do
|
||||
case Registry.lookup(FedSockets.Registry, origin) do
|
||||
[] ->
|
||||
if is_rejected?(origin) do
|
||||
Logger.debug("previously rejected fedsocket requested")
|
||||
{:error, :rejected}
|
||||
else
|
||||
{:error, :missing}
|
||||
end
|
||||
|
||||
[{_pid, %{state: :connected} = socket_info}] ->
|
||||
{:ok, socket_info}
|
||||
|
||||
_ ->
|
||||
{:error, :cache_error}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Retrieves a map of all sockets from the Registry. The keys are the origins and the values are the corresponding SocketInfo
|
||||
"""
|
||||
def list_all do
|
||||
(list_all_connected() ++ list_all_rejected())
|
||||
|> Enum.into(%{})
|
||||
end
|
||||
|
||||
defp list_all_connected do
|
||||
FedSockets.Registry
|
||||
|> Registry.select([{{:"$1", :_, :"$3"}, [], [{{:"$1", :"$3"}}]}])
|
||||
end
|
||||
|
||||
defp list_all_rejected do
|
||||
{:ok, keys} = Cachex.keys(@rejections)
|
||||
|
||||
{:ok, registry_data} =
|
||||
Cachex.execute(@rejections, fn worker ->
|
||||
Enum.map(keys, fn k -> {k, Cachex.get!(worker, k)} end)
|
||||
end)
|
||||
|
||||
registry_data
|
||||
end
|
||||
|
||||
defp clear_prior_rejection(origin),
|
||||
do: Cachex.del(@rejections, origin)
|
||||
|
||||
defp is_rejected?(origin) do
|
||||
case Cachex.get(@rejections, origin) do
|
||||
{:ok, nil} ->
|
||||
false
|
||||
|
||||
{:ok, _} ->
|
||||
true
|
||||
end
|
||||
end
|
||||
|
||||
defp get_or_create_registry_data(origin) do
|
||||
case get_registry_data(origin) do
|
||||
{:error, :missing} ->
|
||||
%SocketInfo{origin: origin}
|
||||
|
||||
{:ok, socket_info} ->
|
||||
socket_info
|
||||
end
|
||||
end
|
||||
|
||||
defp save_registry_data(%SocketInfo{origin: origin, state: :connected} = socket_info) do
|
||||
{:ok, true} = Registry.update_value(FedSockets.Registry, origin, fn _ -> socket_info end)
|
||||
socket_info
|
||||
end
|
||||
|
||||
defp save_registry_data(%SocketInfo{origin: origin, state: :rejected} = socket_info) do
|
||||
rejection_expiration =
|
||||
Pleroma.Config.get([:fed_sockets, :rejection_duration], @default_rejection_duration)
|
||||
|
||||
{:ok, true} = Cachex.put(@rejections, origin, socket_info, ttl: rejection_expiration)
|
||||
socket_info
|
||||
end
|
||||
|
||||
defp set_to_rejected(%SocketInfo{} = socket_info),
|
||||
do: %SocketInfo{socket_info | state: :rejected}
|
||||
end
|
@ -1,137 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.FedSocket do
|
||||
@moduledoc """
|
||||
The FedSocket module abstracts the actions to be taken taken on connections regardless of
|
||||
whether the connection started as inbound or outbound.
|
||||
|
||||
|
||||
Normally outside modules will have no need to call the FedSocket module directly.
|
||||
"""
|
||||
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.Object.Containment
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web.ActivityPub.ObjectView
|
||||
alias Pleroma.Web.ActivityPub.UserView
|
||||
alias Pleroma.Web.ActivityPub.Visibility
|
||||
alias Pleroma.Web.FedSockets.FetchRegistry
|
||||
alias Pleroma.Web.FedSockets.IngesterWorker
|
||||
alias Pleroma.Web.FedSockets.OutgoingHandler
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
|
||||
require Logger
|
||||
|
||||
@shake "61dd18f7-f1e6-49a4-939a-a749fcdc1103"
|
||||
|
||||
def connect_to_host(uri) do
|
||||
case OutgoingHandler.start_link(uri) do
|
||||
{:ok, pid} ->
|
||||
{:ok, pid}
|
||||
|
||||
error ->
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
def close(%SocketInfo{pid: socket_pid}),
|
||||
do: Process.send(socket_pid, :close, [])
|
||||
|
||||
def publish(%SocketInfo{pid: socket_pid}, json) do
|
||||
%{action: :publish, data: json}
|
||||
|> Jason.encode!()
|
||||
|> send_packet(socket_pid)
|
||||
end
|
||||
|
||||
def fetch(%SocketInfo{pid: socket_pid}, id) do
|
||||
fetch_uuid = FetchRegistry.register_fetch(id)
|
||||
|
||||
%{action: :fetch, data: id, uuid: fetch_uuid}
|
||||
|> Jason.encode!()
|
||||
|> send_packet(socket_pid)
|
||||
|
||||
wait_for_fetch_to_return(fetch_uuid, 0)
|
||||
end
|
||||
|
||||
def receive_package(%SocketInfo{} = fed_socket, json) do
|
||||
json
|
||||
|> Jason.decode!()
|
||||
|> process_package(fed_socket)
|
||||
end
|
||||
|
||||
defp wait_for_fetch_to_return(uuid, cntr) do
|
||||
case FetchRegistry.check_fetch(uuid) do
|
||||
{:error, :waiting} ->
|
||||
Process.sleep(:math.pow(cntr, 3) |> Kernel.trunc())
|
||||
wait_for_fetch_to_return(uuid, cntr + 1)
|
||||
|
||||
{:error, :missing} ->
|
||||
Logger.error("FedSocket fetch timed out - #{inspect(uuid)}")
|
||||
{:error, :timeout}
|
||||
|
||||
{:ok, _fr} ->
|
||||
FetchRegistry.pop_fetch(uuid)
|
||||
end
|
||||
end
|
||||
|
||||
defp process_package(%{"action" => "publish", "data" => data}, %{origin: origin} = _fed_socket) do
|
||||
if Containment.contain_origin(origin, data) do
|
||||
IngesterWorker.enqueue("ingest", %{"object" => data})
|
||||
end
|
||||
|
||||
{:reply, %{"action" => "publish_reply", "status" => "processed"}}
|
||||
end
|
||||
|
||||
defp process_package(%{"action" => "fetch_reply", "uuid" => uuid, "data" => data}, _fed_socket) do
|
||||
FetchRegistry.register_fetch_received(uuid, data)
|
||||
{:noreply, nil}
|
||||
end
|
||||
|
||||
defp process_package(%{"action" => "fetch", "uuid" => uuid, "data" => ap_id}, _fed_socket) do
|
||||
{:ok, data} = render_fetched_data(ap_id, uuid)
|
||||
{:reply, data}
|
||||
end
|
||||
|
||||
defp process_package(%{"action" => "publish_reply"}, _fed_socket) do
|
||||
{:noreply, nil}
|
||||
end
|
||||
|
||||
defp process_package(other, _fed_socket) do
|
||||
Logger.warn("unknown json packages received #{inspect(other)}")
|
||||
{:noreply, nil}
|
||||
end
|
||||
|
||||
defp render_fetched_data(ap_id, uuid) do
|
||||
{:ok,
|
||||
%{
|
||||
"action" => "fetch_reply",
|
||||
"status" => "processed",
|
||||
"uuid" => uuid,
|
||||
"data" => represent_item(ap_id)
|
||||
}}
|
||||
end
|
||||
|
||||
defp represent_item(ap_id) do
|
||||
case User.get_by_ap_id(ap_id) do
|
||||
nil ->
|
||||
object = Object.get_cached_by_ap_id(ap_id)
|
||||
|
||||
if Visibility.is_public?(object) do
|
||||
Phoenix.View.render_to_string(ObjectView, "object.json", object: object)
|
||||
else
|
||||
nil
|
||||
end
|
||||
|
||||
user ->
|
||||
Phoenix.View.render_to_string(UserView, "user.json", user: user)
|
||||
end
|
||||
end
|
||||
|
||||
defp send_packet(data, socket_pid) do
|
||||
Process.send(socket_pid, {:send, data}, [])
|
||||
end
|
||||
|
||||
def shake, do: @shake
|
||||
end
|
@ -1,151 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.FetchRegistry do
|
||||
@moduledoc """
|
||||
The FetchRegistry acts as a broker for fetch requests and return values.
|
||||
This allows calling processes to block while waiting for a reply.
|
||||
It doesn't impose it's own process instead using `Cachex` to handle fetches in process, allowing
|
||||
multi threaded processes to avoid bottlenecking.
|
||||
|
||||
Normally outside modules will have no need to call or use the FetchRegistry themselves.
|
||||
|
||||
The `Cachex` parameters can be controlled from the config. Since exact timeout intervals
|
||||
aren't necessary the following settings are used by default:
|
||||
|
||||
config :pleroma, :fed_sockets,
|
||||
fed_socket_fetches: [
|
||||
default: 12_000,
|
||||
interval: 3_000,
|
||||
lazy: false
|
||||
]
|
||||
|
||||
"""
|
||||
|
||||
defmodule FetchRegistryData do
|
||||
defstruct uuid: nil,
|
||||
sent_json: nil,
|
||||
received_json: nil,
|
||||
sent_at: nil,
|
||||
received_at: nil
|
||||
end
|
||||
|
||||
alias Ecto.UUID
|
||||
|
||||
require Logger
|
||||
|
||||
@fetches :fed_socket_fetches
|
||||
|
||||
@doc """
|
||||
Registers a json request wth the FetchRegistry and returns the identifying UUID.
|
||||
"""
|
||||
def register_fetch(json) do
|
||||
%FetchRegistryData{uuid: uuid} =
|
||||
json
|
||||
|> new_registry_data
|
||||
|> save_registry_data
|
||||
|
||||
uuid
|
||||
end
|
||||
|
||||
@doc """
|
||||
Reports on the status of a Fetch given the identifying UUID.
|
||||
|
||||
Will return
|
||||
* {:ok, fetched_object} if a fetch has completed
|
||||
* {:error, :waiting} if a fetch is still pending
|
||||
* {:error, other_error} usually :missing to indicate a fetch that has timed out
|
||||
"""
|
||||
def check_fetch(uuid) do
|
||||
case get_registry_data(uuid) do
|
||||
{:ok, %FetchRegistryData{received_at: nil}} ->
|
||||
{:error, :waiting}
|
||||
|
||||
{:ok, %FetchRegistryData{} = reg_data} ->
|
||||
{:ok, reg_data}
|
||||
|
||||
e ->
|
||||
e
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Retrieves the response to a fetch given the identifying UUID.
|
||||
The completed fetch will be deleted from the FetchRegistry
|
||||
|
||||
Will return
|
||||
* {:ok, fetched_object} if a fetch has completed
|
||||
* {:error, :waiting} if a fetch is still pending
|
||||
* {:error, other_error} usually :missing to indicate a fetch that has timed out
|
||||
"""
|
||||
def pop_fetch(uuid) do
|
||||
case check_fetch(uuid) do
|
||||
{:ok, %FetchRegistryData{received_json: received_json}} ->
|
||||
delete_registry_data(uuid)
|
||||
{:ok, received_json}
|
||||
|
||||
e ->
|
||||
e
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
This is called to register a fetch has returned.
|
||||
It expects the result data along with the UUID that was sent in the request
|
||||
|
||||
Will return the fetched object or :error
|
||||
"""
|
||||
def register_fetch_received(uuid, data) do
|
||||
case get_registry_data(uuid) do
|
||||
{:ok, %FetchRegistryData{received_at: nil} = reg_data} ->
|
||||
reg_data
|
||||
|> set_fetch_received(data)
|
||||
|> save_registry_data()
|
||||
|
||||
{:ok, %FetchRegistryData{} = reg_data} ->
|
||||
Logger.warn("tried to add fetched data twice - #{uuid}")
|
||||
reg_data
|
||||
|
||||
{:error, _} ->
|
||||
Logger.warn("Error adding fetch to registry - #{uuid}")
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp new_registry_data(json) do
|
||||
%FetchRegistryData{
|
||||
uuid: UUID.generate(),
|
||||
sent_json: json,
|
||||
sent_at: :erlang.monotonic_time(:millisecond)
|
||||
}
|
||||
end
|
||||
|
||||
defp get_registry_data(origin) do
|
||||
case Cachex.get(@fetches, origin) do
|
||||
{:ok, nil} ->
|
||||
{:error, :missing}
|
||||
|
||||
{:ok, reg_data} ->
|
||||
{:ok, reg_data}
|
||||
|
||||
_ ->
|
||||
{:error, :cache_error}
|
||||
end
|
||||
end
|
||||
|
||||
defp set_fetch_received(%FetchRegistryData{} = reg_data, data),
|
||||
do: %FetchRegistryData{
|
||||
reg_data
|
||||
| received_at: :erlang.monotonic_time(:millisecond),
|
||||
received_json: data
|
||||
}
|
||||
|
||||
defp save_registry_data(%FetchRegistryData{uuid: uuid} = reg_data) do
|
||||
{:ok, true} = Cachex.put(@fetches, uuid, reg_data)
|
||||
reg_data
|
||||
end
|
||||
|
||||
defp delete_registry_data(origin),
|
||||
do: {:ok, true} = Cachex.del(@fetches, origin)
|
||||
end
|
@ -1,88 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.IncomingHandler do
|
||||
require Logger
|
||||
|
||||
alias Pleroma.Web.FedSockets.FedRegistry
|
||||
alias Pleroma.Web.FedSockets.FedSocket
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
|
||||
import HTTPSignatures, only: [validate_conn: 1, split_signature: 1]
|
||||
|
||||
@behaviour :cowboy_websocket
|
||||
|
||||
def init(req, state) do
|
||||
shake = FedSocket.shake()
|
||||
|
||||
with true <- Pleroma.Config.get([:fed_sockets, :enabled]),
|
||||
sec_protocol <- :cowboy_req.header("sec-websocket-protocol", req, nil),
|
||||
headers = %{"(request-target)" => ^shake} <- :cowboy_req.headers(req),
|
||||
true <- validate_conn(%{req_headers: headers}),
|
||||
%{"keyId" => origin} <- split_signature(headers["signature"]) do
|
||||
req =
|
||||
if is_nil(sec_protocol) do
|
||||
req
|
||||
else
|
||||
:cowboy_req.set_resp_header("sec-websocket-protocol", sec_protocol, req)
|
||||
end
|
||||
|
||||
{:cowboy_websocket, req, %{origin: origin}, %{}}
|
||||
else
|
||||
_ ->
|
||||
{:ok, req, state}
|
||||
end
|
||||
end
|
||||
|
||||
def websocket_init(%{origin: origin}) do
|
||||
case FedRegistry.add_fed_socket(origin) do
|
||||
{:ok, socket_info} ->
|
||||
{:ok, socket_info}
|
||||
|
||||
e ->
|
||||
Logger.error("FedSocket websocket_init failed - #{inspect(e)}")
|
||||
{:error, inspect(e)}
|
||||
end
|
||||
end
|
||||
|
||||
# Use the ping to check if the connection should be expired
|
||||
def websocket_handle(:ping, socket_info) do
|
||||
if SocketInfo.expired?(socket_info) do
|
||||
{:stop, socket_info}
|
||||
else
|
||||
{:ok, socket_info, :hibernate}
|
||||
end
|
||||
end
|
||||
|
||||
def websocket_handle({:text, data}, socket_info) do
|
||||
socket_info = SocketInfo.touch(socket_info)
|
||||
|
||||
case FedSocket.receive_package(socket_info, data) do
|
||||
{:noreply, _} ->
|
||||
{:ok, socket_info}
|
||||
|
||||
{:reply, reply} ->
|
||||
{:reply, {:text, Jason.encode!(reply)}, socket_info}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("incoming error - receive_package: #{inspect(reason)}")
|
||||
{:ok, socket_info}
|
||||
end
|
||||
end
|
||||
|
||||
def websocket_info({:send, message}, socket_info) do
|
||||
socket_info = SocketInfo.touch(socket_info)
|
||||
|
||||
{:reply, {:text, message}, socket_info}
|
||||
end
|
||||
|
||||
def websocket_info(:close, state) do
|
||||
{:stop, state}
|
||||
end
|
||||
|
||||
def websocket_info(message, state) do
|
||||
Logger.debug("#{__MODULE__} unknown message #{inspect(message)}")
|
||||
{:ok, state}
|
||||
end
|
||||
end
|
@ -1,33 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.IngesterWorker do
|
||||
use Pleroma.Workers.WorkerHelper, queue: "ingestion_queue"
|
||||
require Logger
|
||||
|
||||
alias Pleroma.Web.Federator
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Job{args: %{"op" => "ingest", "object" => ingestee}}) do
|
||||
try do
|
||||
ingestee
|
||||
|> Jason.decode!()
|
||||
|> do_ingestion()
|
||||
rescue
|
||||
e ->
|
||||
Logger.error("IngesterWorker error - #{inspect(e)}")
|
||||
e
|
||||
end
|
||||
end
|
||||
|
||||
defp do_ingestion(params) do
|
||||
case Federator.incoming_ap_doc(params) do
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
{:ok, object} ->
|
||||
{:ok, object}
|
||||
end
|
||||
end
|
||||
end
|
@ -1,151 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.OutgoingHandler do
|
||||
use GenServer
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pleroma.Application
|
||||
alias Pleroma.Web.ActivityPub.InternalFetchActor
|
||||
alias Pleroma.Web.FedSockets
|
||||
alias Pleroma.Web.FedSockets.FedRegistry
|
||||
alias Pleroma.Web.FedSockets.FedSocket
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
|
||||
def start_link(uri) do
|
||||
GenServer.start_link(__MODULE__, %{uri: uri})
|
||||
end
|
||||
|
||||
def init(%{uri: uri}) do
|
||||
case initiate_connection(uri) do
|
||||
{:ok, ws_origin, conn_pid} ->
|
||||
FedRegistry.add_fed_socket(ws_origin, conn_pid)
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.debug("Outgoing connection failed - #{inspect(reason)}")
|
||||
:ignore
|
||||
end
|
||||
end
|
||||
|
||||
def handle_info({:gun_ws, conn_pid, _ref, {:text, data}}, socket_info) do
|
||||
socket_info = SocketInfo.touch(socket_info)
|
||||
|
||||
case FedSocket.receive_package(socket_info, data) do
|
||||
{:noreply, _} ->
|
||||
{:noreply, socket_info}
|
||||
|
||||
{:reply, reply} ->
|
||||
:gun.ws_send(conn_pid, {:text, Jason.encode!(reply)})
|
||||
{:noreply, socket_info}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("incoming error - receive_package: #{inspect(reason)}")
|
||||
{:noreply, socket_info}
|
||||
end
|
||||
end
|
||||
|
||||
def handle_info(:close, state) do
|
||||
Logger.debug("Sending close frame !!!!!!!")
|
||||
{:close, state}
|
||||
end
|
||||
|
||||
def handle_info({:gun_down, _pid, _prot, :closed, _}, state) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
def handle_info({:send, data}, %{conn_pid: conn_pid} = socket_info) do
|
||||
socket_info = SocketInfo.touch(socket_info)
|
||||
:gun.ws_send(conn_pid, {:text, data})
|
||||
{:noreply, socket_info}
|
||||
end
|
||||
|
||||
def handle_info({:gun_ws, _, _, :pong}, state) do
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
|
||||
def handle_info(msg, state) do
|
||||
Logger.debug("#{__MODULE__} unhandled event #{inspect(msg)}")
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
def terminate(reason, state) do
|
||||
Logger.debug(
|
||||
"#{__MODULE__} terminating outgoing connection for #{inspect(state)} for #{inspect(reason)}"
|
||||
)
|
||||
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def initiate_connection(uri) do
|
||||
ws_uri =
|
||||
uri
|
||||
|> SocketInfo.origin()
|
||||
|> FedSockets.uri_for_origin()
|
||||
|
||||
%{host: host, port: port, path: path} = URI.parse(ws_uri)
|
||||
|
||||
with {:ok, conn_pid} <- :gun.open(to_charlist(host), port, %{protocols: [:http]}),
|
||||
{:ok, _} <- :gun.await_up(conn_pid),
|
||||
reference <-
|
||||
:gun.get(conn_pid, to_charlist(path), [
|
||||
{'user-agent', to_charlist(Application.user_agent())}
|
||||
]),
|
||||
{:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
|
||||
headers <- build_headers(uri),
|
||||
ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
|
||||
receive do
|
||||
{:gun_upgrade, ^conn_pid, ^ref, [<<"websocket">>], _} ->
|
||||
{:ok, ws_uri, conn_pid}
|
||||
after
|
||||
15_000 ->
|
||||
Logger.debug("Fedsocket timeout connecting to #{inspect(uri)}")
|
||||
{:error, :timeout}
|
||||
end
|
||||
else
|
||||
{:response, :nofin, 404, _} ->
|
||||
{:error, :fedsockets_not_supported}
|
||||
|
||||
e ->
|
||||
Logger.debug("Fedsocket error connecting to #{inspect(uri)}")
|
||||
{:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
defp build_headers(uri) do
|
||||
host_for_sig = uri |> URI.parse() |> host_signature()
|
||||
|
||||
shake = FedSocket.shake()
|
||||
digest = "SHA-256=" <> (:crypto.hash(:sha256, shake) |> Base.encode64())
|
||||
date = Pleroma.Signature.signed_date()
|
||||
shake_size = byte_size(shake)
|
||||
|
||||
signature_opts = %{
|
||||
"(request-target)": shake,
|
||||
"content-length": to_charlist("#{shake_size}"),
|
||||
date: date,
|
||||
digest: digest,
|
||||
host: host_for_sig
|
||||
}
|
||||
|
||||
signature = Pleroma.Signature.sign(InternalFetchActor.get_actor(), signature_opts)
|
||||
|
||||
[
|
||||
{'signature', to_charlist(signature)},
|
||||
{'date', date},
|
||||
{'digest', to_charlist(digest)},
|
||||
{'content-length', to_charlist("#{shake_size}")},
|
||||
{to_charlist("(request-target)"), to_charlist(shake)},
|
||||
{'user-agent', to_charlist(Application.user_agent())}
|
||||
]
|
||||
end
|
||||
|
||||
defp host_signature(%{host: host, scheme: scheme, port: port}) do
|
||||
if port == URI.default_port(scheme) do
|
||||
host
|
||||
else
|
||||
"#{host}:#{port}"
|
||||
end
|
||||
end
|
||||
end
|
@ -1,52 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.SocketInfo do
|
||||
defstruct origin: nil,
|
||||
pid: nil,
|
||||
conn_pid: nil,
|
||||
state: :default,
|
||||
connected_until: nil
|
||||
|
||||
alias Pleroma.Web.FedSockets.SocketInfo
|
||||
@default_connection_duration 15 * 60 * 1000
|
||||
|
||||
def build(uri, conn_pid \\ nil) do
|
||||
uri
|
||||
|> build_origin()
|
||||
|> build_pids(conn_pid)
|
||||
|> touch()
|
||||
end
|
||||
|
||||
def touch(%SocketInfo{} = socket_info),
|
||||
do: %{socket_info | connected_until: new_ttl()}
|
||||
|
||||
def connect(%SocketInfo{} = socket_info),
|
||||
do: %{socket_info | state: :connected}
|
||||
|
||||
def expired?(%{connected_until: connected_until}),
|
||||
do: connected_until < :erlang.monotonic_time(:millisecond)
|
||||
|
||||
def origin(uri),
|
||||
do: build_origin(uri).origin
|
||||
|
||||
defp build_pids(socket_info, conn_pid),
|
||||
do: struct(socket_info, pid: self(), conn_pid: conn_pid)
|
||||
|
||||
defp build_origin(uri) when is_binary(uri),
|
||||
do: uri |> URI.parse() |> build_origin
|
||||
|
||||
defp build_origin(%{host: host, port: nil, scheme: scheme}),
|
||||
do: build_origin(%{host: host, port: URI.default_port(scheme)})
|
||||
|
||||
defp build_origin(%{host: host, port: port}),
|
||||
do: %SocketInfo{origin: "#{host}:#{port}"}
|
||||
|
||||
defp new_ttl do
|
||||
connection_duration =
|
||||
Pleroma.Config.get([:fed_sockets, :connection_duration], @default_connection_duration)
|
||||
|
||||
:erlang.monotonic_time(:millisecond) + connection_duration
|
||||
end
|
||||
end
|
@ -1,59 +0,0 @@
|
||||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.FedSockets.Supervisor do
|
||||
use Supervisor
|
||||
import Cachex.Spec
|
||||
|
||||
def start_link(opts) do
|
||||
Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
|
||||
end
|
||||
|
||||
def init(args) do
|
||||
children = [
|
||||
build_cache(:fed_socket_fetches, args),
|
||||
build_cache(:fed_socket_rejections, args),
|
||||
{Registry, keys: :unique, name: FedSockets.Registry, meta: [rejected: %{}]}
|
||||
]
|
||||
|
||||
opts = [strategy: :one_for_all, name: Pleroma.Web.Streamer.Supervisor]
|
||||
Supervisor.init(children, opts)
|
||||
end
|
||||
|
||||
defp build_cache(name, args) do
|
||||
opts = get_opts(name, args)
|
||||
|
||||
%{
|
||||
id: String.to_atom("#{name}_cache"),
|
||||
start: {Cachex, :start_link, [name, opts]},
|
||||
type: :worker
|
||||
}
|
||||
end
|
||||
|
||||
defp get_opts(cache_name, args)
|
||||
when cache_name in [:fed_socket_fetches, :fed_socket_rejections] do
|
||||
default = get_opts_or_config(args, cache_name, :default, 15_000)
|
||||
interval = get_opts_or_config(args, cache_name, :interval, 3_000)
|
||||
lazy = get_opts_or_config(args, cache_name, :lazy, false)
|
||||
|
||||
[expiration: expiration(default: default, interval: interval, lazy: lazy)]
|
||||
end
|
||||
|
||||
defp get_opts(name, args) do
|
||||
Keyword.get(args, name, [])
|
||||
end
|
||||
|
||||
defp get_opts_or_config(args, name, key, default) do
|
||||
args
|
||||
|> Keyword.get(name, [])
|
||||
|> Keyword.get(key)
|
||||
|> case do
|
||||
nil ->
|
||||
Pleroma.Config.get([:fed_sockets, name, key], default)
|
||||
|
||||
value ->
|
||||
value
|
||||
end
|
||||
end
|
||||
end
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue