mirror_maker/fedi-block-api
1
0
Fork 0
mirror of https://git.kiwifarms.net/mint/fedi-block-api.git synced 2024-10-22 21:39:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

Track fetch date in DB & correctly prepend newly added descriptions

Browse source
This commit is contained in:
Mint 2022-11-07 23:33:23 +03:00
parent 03358a3c6e
commit cedee03bba
4 changed files with 56 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
README.md
View file

@ -12,10 +12,11 @@ Used to see which instances block yours.
sudo useradd -m fba sudo useradd -m fba
sudo mkdir -p /opt/fedi-block-api sudo mkdir -p /opt/fedi-block-api
sudo chown -R fba:fba /opt/fedi-block-api sudo chown -R fba:fba /opt/fedi-block-api
sudo -Hu fba git clone https://gitlab.com/EnjuAihara/fedi-block-api.git /opt/fedi-block-api sudo -Hu fba git clone https://git.kiwifarms.net/mint/fedi-block-api.git /opt/fedi-block-api
cd /opt/fedi-block-api cd /opt/fedi-block-api
sudo -Hu fba pip3 install -r requirements.txt sudo -Hu fba pip3 install -r requirements.txt
sudo -Hu fba cp blocks_preloaded.db blocks.db sudo -Hu fba cp blocks_empty.db blocks.db
sudo -Hu fba python3 fetch_instances.py mastodon.social # try a bunch of large servers here
sudo -Hu fba cp config.defaults.json config.json sudo -Hu fba cp config.defaults.json config.json
``` ```

BIN
blocks_empty.db Normal file
View file

Binary file not shown.

BIN
blocks_preloaded.db
View file

Binary file not shown.

71
fetch_blocks.py
View file

@ -5,6 +5,7 @@ import sqlite3
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from json import dumps from json import dumps
import re import re
from time import time
headers = { headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0" "user-agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"
@ -195,7 +196,8 @@ conn = sqlite3.connect("blocks.db")
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"select domain, software from instances where software in ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')" # "select domain, software from instances where software in ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial')"
"select domain, software from instances where domain = 'glaceon.social'"
) )
for blocker, software in c.fetchall(): for blocker, software in c.fetchall():
@ -233,14 +235,20 @@ for blocker, software in c.fetchall():
"insert into instances select ?, ?, ?", "insert into instances select ?, ?, ?",
(blocked, get_hash(blocked), get_type(blocked)), (blocked, get_hash(blocked), get_type(blocked)),
) )
timestamp = int(time())
c.execute( c.execute(
"select * from blocks where blocker = ? and blocked = ? and block_level = ?", "select * from blocks where blocker = ? and blocked = ? and block_level = ?",
(blocker, blocked, block_level), (blocker, blocked, block_level),
) )
if c.fetchone() == None: if c.fetchone() == None:
c.execute( c.execute(
"insert into blocks select ?, ?, '', ?", "insert into blocks select ?, ?, '', ?, ?, ?",
(blocker, blocked, block_level), (blocker, blocked, block_level, timestamp, timestamp),
)
else:
c.execute(
"update blocks set last_seen = ? where blocker = ? and blocked = ? and block_level = ?",
(timestamp, blocker, blocked, block_level)
) )
conn.commit() conn.commit()
# Reasons # Reasons
@ -264,7 +272,7 @@ for blocker, software in c.fetchall():
if searchres != None: if searchres != None:
blocked = searchres[0] blocked = searchres[0]
c.execute( c.execute(
"update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?", "update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ? and reason = ''",
(reason["reason"], blocker, blocked, block_level), (reason["reason"], blocker, blocked, block_level),
) )
conn.commit() conn.commit()
@ -319,20 +327,33 @@ for blocker, software in c.fetchall():
if searchres != None: if searchres != None:
blocked = searchres[0] blocked = searchres[0]
timestamp = int(time())
c.execute( c.execute(
"select * from blocks where blocker = ? and blocked = ? and block_level = ?", "select * from blocks where blocker = ? and blocked = ? and block_level = ?",
(blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level), (blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level),
) )
if c.fetchone() == None: if c.fetchone() == None:
c.execute( c.execute(
"insert into blocks select ?, ?, ?, ?", "insert into blocks select ?, ?, ?, ?, ?, ?",
( (
blocker, blocker,
blocked if blocked.count("*") <= 1 else blocked_hash, blocked if blocked.count("*") <= 1 else blocked_hash,
reason, reason,
block_level, block_level,
timestamp,
timestamp,
), ),
) )
else:
c.execute(
"update blocks set last_seen = ? where blocker = ? and blocked = ? and block_level = ?",
(timestamp, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level),
)
if reason != '':
c.execute(
"update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ? and reason = ''",
(reason, blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level),
)
conn.commit() conn.commit()
except Exception as e: except Exception as e:
print("error:", e, blocker) print("error:", e, blocker)
@ -374,20 +395,34 @@ for blocker, software in c.fetchall():
"insert into instances select ?, ?, ?", "insert into instances select ?, ?, ?",
(blocked, get_hash(blocked), get_type(blocked)), (blocked, get_hash(blocked), get_type(blocked)),
) )
timestamp = int(time())
c.execute( c.execute(
"select * from blocks where blocker = ? and blocked = ?", "select * from blocks where blocker = ? and blocked = ? and reason = ?",
(blocker, blocked), (blocker, blocked, reason),
) )
if c.fetchone() == None: if c.fetchone() == None:
c.execute( c.execute(
"insert into blocks select ?, ?, ?, ?", "insert into blocks select ?, ?, ?, ?, ?, ?",
( (
blocker, blocker,
blocked, blocked,
reason, reason,
block_level, block_level,
timestamp,
timestamp
), ),
) )
else:
c.execute(
"update blocks set last_seen = ? where blocker = ? and blocked = ? and block_level = ?",
(timestamp, blocker, blocked, block_level),
)
if reason != '':
c.execute(
"update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ? and reason = ''",
(reason, blocker, blocked, block_level),
)
conn.commit() conn.commit()
except Exception as e: except Exception as e:
print("error:", e, blocker) print("error:", e, blocker)
@ -422,23 +457,23 @@ for blocker, software in c.fetchall():
"select * from blocks where blocker = ? and blocked = ? and block_level = ?", "select * from blocks where blocker = ? and blocked = ? and block_level = ?",
(blocker, blocked, "reject"), (blocker, blocked, "reject"),
) )
timestamp = int(time())
if c.fetchone() == None: if c.fetchone() == None:
c.execute( c.execute(
"insert into blocks select ?, ?, ?, ?", "insert into blocks select ?, ?, ?, ?, ?, ?",
(blocker, blocked, "", "reject"), (blocker, blocked, "", "reject", timestamp, timestamp),
)
else:
c.execute(
"update blocks set last_seen = ? where blocker = ? and blocked = ? and block_level = ?",
(timestamp, blocker, blocked, "reject"),
) )
if "public_comment" in peer: if "public_comment" in peer:
reason = peer["public_comment"] reason = peer["public_comment"]
c.execute( c.execute(
"select * from blocks where blocker = ? and blocked = ? and reason != ? and block_level = ?", "update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ? and reason = ''",
(blocker, blocked, "", "reject"), (reason, blocker, blocked, "reject"),
) )
if c.fetchone() == None:
c.execute(
"update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?",
(reason, blocker, blocked, "reject"),
)
conn.commit() conn.commit()
except Exception as e: except Exception as e:
print("error:", e, blocker) print("error:", e, blocker)