2022-03-03 23:41:18 +00:00
from requests import get
2022-03-05 10:56:13 +00:00
from hashlib import sha256
2022-03-03 23:41:18 +00:00
import sqlite3
2022-03-31 17:00:08 +00:00
from bs4 import BeautifulSoup
2022-04-06 15:40:49 +00:00
headers = {
" user-agent " : " fedi-block-api (https://gitlab.com/EnjuAihara/fedi-block-api) "
}
2022-03-31 17:00:08 +00:00
def get_mastodon_blocks ( domain : str ) - > dict :
2022-04-08 21:12:01 +00:00
blocks = {
" Suspended servers " : [ ] ,
" Filtered media " : [ ] ,
" Limited servers " : [ ] ,
" Silenced servers " : [ ] ,
}
2022-03-31 17:00:08 +00:00
2022-04-08 21:12:01 +00:00
try :
doc = BeautifulSoup (
get ( f " https:// { domain } /about/more " , headers = headers , timeout = 5 ) . text ,
" html.parser " ,
)
except :
return { }
for header in doc . find_all ( " h3 " ) :
for line in header . find_next_siblings ( " table " ) [ 0 ] . find_all ( " tr " ) [ 1 : ] :
if header . text in blocks :
blocks [ header . text ] . append (
{
" domain " : line . find ( " span " ) . text ,
" hash " : line . find ( " span " ) [ " title " ] [ 9 : ] ,
" reason " : line . find_all ( " td " ) [ 1 ] . text . strip ( ) ,
}
)
return {
" reject " : blocks [ " Suspended servers " ] ,
" media_removal " : blocks [ " Filtered media " ] ,
" federated_timeline_removal " : blocks [ " Limited servers " ] + blocks [ " Silenced servers " ] ,
}
2022-03-31 17:00:08 +00:00
2022-03-03 23:41:18 +00:00
2022-03-31 15:17:11 +00:00
def get_type ( domain : str ) - > str :
2022-03-21 13:04:44 +00:00
try :
2022-04-08 20:07:05 +00:00
res = get ( f " https:// { domain } /nodeinfo/2.1.json " , headers = headers , timeout = 5 )
if res . status_code == 404 :
res = get ( f " https:// { domain } /nodeinfo/2.0.json " , headers = headers , timeout = 5 )
if res . ok :
return res . json ( ) [ " software " ] [ " name " ]
elif res . status_code == 404 :
res = get ( f " https:// { domain } /api/v1/instance " , headers = headers , timeout = 5 )
if res . ok :
2022-03-21 13:04:44 +00:00
return " mastodon "
2022-04-08 20:07:05 +00:00
except :
return None
2022-03-21 13:04:44 +00:00
2022-03-03 23:41:18 +00:00
conn = sqlite3 . connect ( " blocks.db " )
c = conn . cursor ( )
2022-04-07 11:14:25 +00:00
c . execute ( " select domain, software from instances where software in ( ' pleroma ' , ' mastodon ' ) " )
2022-03-31 17:00:08 +00:00
2022-03-31 22:58:52 +00:00
for blocker , software in c . fetchall ( ) :
if software == " pleroma " :
2022-03-03 23:41:18 +00:00
print ( blocker )
try :
2022-03-21 13:04:44 +00:00
# Blocks
2022-03-05 08:41:11 +00:00
c . execute ( " delete from blocks where blocker = ? " , ( blocker , ) )
2022-04-06 15:40:49 +00:00
json = get ( f " https:// { blocker } /nodeinfo/2.1.json " , headers = headers , timeout = 5 ) . json ( )
2022-03-31 16:07:32 +00:00
if " mrf_simple " in json [ " metadata " ] [ " federation " ] :
for mrf in json [ " metadata " ] [ " federation " ] [ " mrf_simple " ] :
for blocked in json [ " metadata " ] [ " federation " ] [ " mrf_simple " ] [ mrf ] :
if blocked == " " :
continue
2022-03-31 22:32:38 +00:00
c . execute ( " select domain from instances where domain = ? " , ( blocked , ) )
if c . fetchone ( ) == None :
2022-03-31 16:07:32 +00:00
c . execute ( " insert into instances select ?, ?, ? " , ( blocked , sha256 ( bytes ( blocked , " utf-8 " ) ) . hexdigest ( ) , get_type ( blocked ) ) )
c . execute ( " insert into blocks select ?, ?, ' ' , ? " , ( blocker , blocked , mrf ) )
# Quarantined Instances
if " quarantined_instances " in json [ " metadata " ] [ " federation " ] :
for blocked in json [ " metadata " ] [ " federation " ] [ " quarantined_instances " ] :
2022-03-05 12:09:00 +00:00
if blocked == " " :
continue
2022-03-31 22:32:38 +00:00
c . execute ( " select domain from instances where domain = ? " , ( blocked , ) )
if c . fetchone ( ) == None :
2022-03-21 13:04:44 +00:00
c . execute ( " insert into instances select ?, ?, ? " , ( blocked , sha256 ( bytes ( blocked , " utf-8 " ) ) . hexdigest ( ) , get_type ( blocked ) ) )
2022-03-31 16:07:32 +00:00
c . execute ( " insert into blocks select ?, ?, ' ' , ' quarantined_instances ' " , ( blocker , blocked ) )
2022-03-03 23:41:18 +00:00
conn . commit ( )
2022-03-21 13:04:44 +00:00
# Reasons
2022-03-31 16:07:32 +00:00
if " mrf_simple_info " in json [ " metadata " ] [ " federation " ] :
for mrf in json [ " metadata " ] [ " federation " ] [ " mrf_simple_info " ] :
for blocked in json [ " metadata " ] [ " federation " ] [ " mrf_simple_info " ] [ mrf ] :
c . execute ( " update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ? " , ( json [ " metadata " ] [ " federation " ] [ " mrf_simple_info " ] [ mrf ] [ blocked ] [ " reason " ] , blocker , blocked , mrf ) )
if " quarantined_instances_info " in json [ " metadata " ] [ " federation " ] :
for blocked in json [ " metadata " ] [ " federation " ] [ " quarantined_instances_info " ] [ " quarantined_instances " ] :
c . execute ( " update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ' quarantined_instances ' " , ( json [ " metadata " ] [ " federation " ] [ " quarantined_instances_info " ] [ " quarantined_instances " ] [ blocked ] [ " reason " ] , blocker , blocked ) )
2022-03-05 08:21:10 +00:00
conn . commit ( )
2022-03-21 13:04:44 +00:00
except Exception as e :
print ( " error: " , e , blocker )
2022-03-31 22:58:52 +00:00
elif software == " mastodon " :
2022-03-04 15:50:32 +00:00
print ( blocker )
try :
2022-03-05 08:41:11 +00:00
c . execute ( " delete from blocks where blocker = ? " , ( blocker , ) )
2022-03-31 17:00:08 +00:00
json = get_mastodon_blocks ( blocker )
2022-03-31 22:58:52 +00:00
for block_level in json :
2022-03-05 11:19:28 +00:00
for blocked in json [ block_level ] :
if blocked [ " domain " ] . count ( " * " ) > 1 :
2022-03-31 22:58:52 +00:00
# instance is censored, check if domain of hash is known, if not, insert the hash
2022-03-05 11:19:28 +00:00
c . execute ( " insert into blocks select ?, ifnull((select domain from instances where hash = ?), ?), ?, ? " , ( blocker , blocked [ " hash " ] , blocked [ " hash " ] , blocked [ ' reason ' ] , block_level ) )
else :
2022-03-31 22:58:52 +00:00
# instance is not censored
2022-03-31 22:32:38 +00:00
c . execute ( " select domain from instances where domain = ? " , ( blocked [ " domain " ] , ) )
if c . fetchone ( ) == None :
2022-03-31 22:58:52 +00:00
# if instance not known, add it
2022-03-21 13:04:44 +00:00
c . execute ( " insert into instances select ?, ?, ? " , ( blocked [ " domain " ] , sha256 ( bytes ( blocked [ " domain " ] , " utf-8 " ) ) . hexdigest ( ) , get_type ( blocked [ " domain " ] ) ) )
2022-03-05 11:19:28 +00:00
c . execute ( " insert into blocks select ?, ?, ?, ? " , ( blocker , blocked [ " domain " ] , blocked [ " reason " ] , block_level ) )
2022-03-04 15:50:32 +00:00
conn . commit ( )
2022-03-21 13:04:44 +00:00
except Exception as e :
print ( " error: " , e , blocker )
2022-03-31 17:00:08 +00:00
conn . close ( )