[feature] Add experimental instance-federation-spam-filter option (#2685)

* [chore] Move `visibility` to `filter/visibility`

* [feature] Add experimental instance-federation-spam-filter option

cmd/gotosocial/action/server/server.go

@@ -32,13 +32,14 @@
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/spam"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/metrics"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
 	tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"go.uber.org/automaxprocs/maxprocs"
 
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -143,8 +144,9 @@ func(context.Context, time.Time) {
 	mediaManager := media.NewManager(&state)
 	oauthServer := oauth.New(ctx, dbService)
 	typeConverter := typeutils.NewConverter(&state)
-	filter := visibility.NewFilter(&state)
+	visFilter := visibility.NewFilter(&state)
-	federatingDB := federatingdb.New(&state, typeConverter, filter)
+	spamFilter := spam.NewFilter(&state)
+	federatingDB := federatingdb.New(&state, typeConverter, visFilter, spamFilter)
 	transportController := transport.NewController(&state, federatingDB, &federation.Clock{}, client)
 	federator := federation.NewFederator(&state, federatingDB, transportController, typeConverter, mediaManager)
 
@@ -168,7 +170,7 @@ func(context.Context, time.Time) {
 	// Initialize timelines.
 	state.Timelines.Home = timeline.NewManager(
 		tlprocessor.HomeTimelineGrab(&state),
-		tlprocessor.HomeTimelineFilter(&state, filter),
+		tlprocessor.HomeTimelineFilter(&state, visFilter),
 		tlprocessor.HomeTimelineStatusPrepare(&state, typeConverter),
 		tlprocessor.SkipInsert(),
 	)
@@ -178,7 +180,7 @@ func(context.Context, time.Time) {
 
 	state.Timelines.List = timeline.NewManager(
 		tlprocessor.ListTimelineGrab(&state),
-		tlprocessor.ListTimelineFilter(&state, filter),
+		tlprocessor.ListTimelineFilter(&state, visFilter),
 		tlprocessor.ListTimelineStatusPrepare(&state, typeConverter),
 		tlprocessor.SkipInsert(),
 	)

cmd/gotosocial/action/testrig/testrig.go

@@ -34,6 +34,7 @@
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/language"
@@ -47,7 +48,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/web"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )

docs/admin/spam.md

@@ -0,0 +1,23 @@
+# Spam Filtering
+
+To make life a bit easier for admins trying to combat spam messages from open signup instances, GoToSocial has an experimental spam filter option.
+
+If you or your users are being barraged by spam, try setting the option `instance-federation-spam-filter` to true in your config.yaml. You can read more about the heuristics used in the [instance config page](../configuration/instance.md).
+
+Messages that are considered to be spam will not be stored on your instance, and will not generate notifications.
+
+!!! warning
+    Spam filters are necessarily imperfect tools, since they will likely catch at least a few legitimate messages in the filter, or indeed fail to catch some messages that *are* spam.
+    
+    Enabling `instance-federation-spam-filter` should be viewed as a "battening down the hatches" option for when the fediverse is facing a spam wave. Under normal circumstances, you will likely want to leave it turned off to avoid filtering out legitimate messages by accident.
+
+!!! tip
+    If you want to check what's being caught by the spam filter (if anything), grep your logs for the phrase "looked like spam".
+    
+    If you're [running GoToSocial as a systemd service](../getting_started/installation/metal.md#optional-enable-the-systemd-service), you can do this with the command:
+    
+    ```bash
+    journalctl -u gotosocial --no-pager | grep 'looked like spam'
+    ```
+    
+    If you see no output, that means no spam has been caught in the filter. Otherwise, you will see one or more log lines with links to statuses that have been filtered and dropped.

docs/configuration/instance.md

@@ -39,6 +39,43 @@ instance-languages: []
 # Default: "blocklist"
 instance-federation-mode: "blocklist"
 
+# Bool. Enable spam filtering heuristics for messages entering your instance
+# via the federation API. Regardless of what you set here, basic checks
+# for message relevancy will still be performed, but you can try enabling
+# this setting if you are being spammed with unwanted messages from other
+# instances, and want to more strictly filter out spam messages.
+#
+# THIS IS CURRENTLY AN EXPERIMENTAL SETTING, AND MAY FILTER OUT LEGITIMATE
+# MESSAGES, OR FAIL TO FILTER OUT SPAMMY MESSAGES. It is recommended to
+# only enable this setting when the fediverse is in the midst of a spam
+# wave, and you need to batten down the hatches to keep your instance usable.
+#
+# The decision of whether a message counts as spam or not is made based on
+# the following heuristics, in order, where receiver = the account on your
+# instance that received a message in their inbox, and requester = the
+# account on a remote instance that sent the message.
+#
+# First, basic relevancy checks
+#
+#  1. Receiver follows requester. Return OK.
+#  2. Statusable doesn't mention receiver. Return NotRelevant.
+#
+# If instance-federation-spam-filter = false, then return OK now.
+# Otherwise check:
+#
+#  3. Receiver is locked and is followed by requester. Return OK.
+#  4. Five or more people are mentioned. Return Spam.
+#  5. Receiver follow (requests) a mentioned account. Return OK.
+#  6. Statusable has a media attachment. Return Spam.
+#  7. Statusable contains non-mention, non-hashtag links. Return Spam.
+#
+# Messages identified as spam will be dropped from your instance, and not
+# inserted into the database, or into home timelines or notifications.
+#
+# Options: [true, false]
+# Default: false
+instance-federation-spam-filter: false
+
 # Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=open in order
 # to see a list of instances that this instance 'peers' with. Even if set to 'false', then authenticated
 # users (members of the instance) will still be able to query the endpoint.

example/config.yaml

@@ -309,6 +309,43 @@ instance-languages: []
 # Default: "blocklist"
 instance-federation-mode: "blocklist"
 
+# Bool. Enable spam filtering heuristics for messages entering your instance
+# via the federation API. Regardless of what you set here, basic checks
+# for message relevancy will still be performed, but you can try enabling
+# this setting if you are being spammed with unwanted messages from other
+# instances, and want to more strictly filter out spam messages.
+#
+# THIS IS CURRENTLY AN EXPERIMENTAL SETTING, AND MAY FILTER OUT LEGITIMATE
+# MESSAGES, OR FAIL TO FILTER OUT SPAMMY MESSAGES. It is recommended to
+# only enable this setting when the fediverse is in the midst of a spam
+# wave, and you need to batten down the hatches to keep your instance usable.
+#
+# The decision of whether a message counts as spam or not is made based on
+# the following heuristics, in order, where receiver = the account on your
+# instance that received a message in their inbox, and requester = the
+# account on a remote instance that sent the message.
+#
+# First, basic relevancy checks
+#
+#  1. Receiver follows requester. Return OK.
+#  2. Statusable doesn't mention receiver. Return NotRelevant.
+#
+# If instance-federation-spam-filter = false, then return OK now.
+# Otherwise check:
+#
+#  3. Receiver is locked and is followed by requester. Return OK.
+#  4. Five or more people are mentioned. Return Spam.
+#  5. Receiver follow (requests) a mentioned account. Return OK.
+#  6. Statusable has a media attachment. Return Spam.
+#  7. Statusable contains non-mention, non-hashtag links. Return Spam.
+#
+# Messages identified as spam will be dropped from your instance, and not
+# inserted into the database, or into home timelines or notifications.
+#
+# Options: [true, false]
+# Default: false
+instance-federation-spam-filter: false
+
 # Bool. Allow unauthenticated users to make queries to /api/v1/instance/peers?filter=open in order
 # to see a list of instances that this instance 'peers' with. Even if set to 'false', then authenticated
 # users (members of the instance) will still be able to query the endpoint.

internal/ap/extract.go

@@ -33,6 +33,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // ExtractObjects will extract object vocab.Types from given implementing interface.
@@ -776,11 +777,21 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 	}
 	tagName := strings.TrimPrefix(name, "#")
 
-	yeah := func() *bool { t := true; return &t }
+	// Extract href for the tag, if set.
+	//
+	// Fine if not, it's only used for spam
+	// checking anyway so not critical.
+	var href string
+	hrefProp := i.GetActivityStreamsHref()
+	if hrefProp != nil && hrefProp.IsIRI() {
+		href = hrefProp.GetIRI().String()
+	}
+
 	return &gtsmodel.Tag{
 		Name:     tagName,
-		Useable:  yeah(), // Assume true by default.
+		Useable:  util.Ptr(true), // Assume true by default.
-		Listable: yeah(), // Assume true by default.
+		Listable: util.Ptr(true), // Assume true by default.
+		Href:     href,
 	}, nil
 }
 

internal/api/activitypub/emoji/emojiget_test.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
@@ -36,7 +37,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/activitypub/users/user_test.go

@@ -24,6 +24,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
@@ -31,7 +32,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/accounts/account_test.go

@@ -30,6 +30,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -37,7 +38,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/admin/admin_test.go

@@ -30,6 +30,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -37,7 +38,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/bookmarks/bookmarks_test.go

@@ -35,6 +35,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -42,7 +43,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/favourites/favourites_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/followrequests/followrequest_test.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -36,7 +37,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/instance/instance_test.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -36,7 +37,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/lists/lists_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/media/mediacreate_test.go

@@ -37,6 +37,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -45,7 +46,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/media/mediaupdate_test.go

@@ -34,6 +34,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -42,7 +43,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/polls/polls_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/reports/reports_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/search/search_test.go

@@ -29,6 +29,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -36,7 +37,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/statuses/status_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/streaming/streaming_test.go

@@ -35,6 +35,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -42,7 +43,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/client/user/user_test.go

@@ -23,13 +23,13 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/fileserver/fileserver_test.go

@@ -23,6 +23,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -31,7 +32,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/api/wellknown/webfinger/webfinger_test.go

@@ -23,6 +23,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -30,7 +31,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/cleaner/media_test.go

@@ -28,6 +28,7 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -35,7 +36,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/config/config.go

@@ -78,6 +78,7 @@ type Configuration struct {
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
 
 	InstanceFederationMode         string             `name:"instance-federation-mode" usage:"Set instance federation mode."`
+	InstanceFederationSpamFilter   bool               `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
 	InstanceExposePeers            bool               `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
 	InstanceExposeSuspended        bool               `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
 	InstanceExposeSuspendedWeb     bool               `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`

internal/config/defaults.go

@@ -59,6 +59,7 @@
 	WebAssetBaseDir:    "./web/assets/",
 
 	InstanceFederationMode:         InstanceFederationModeDefault,
+	InstanceFederationSpamFilter:   false,
 	InstanceExposePeers:            false,
 	InstanceExposeSuspended:        false,
 	InstanceExposeSuspendedWeb:     false,

internal/config/flags.go

@@ -84,6 +84,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 
 		// Instance
 		cmd.Flags().String(InstanceFederationModeFlag(), cfg.InstanceFederationMode, fieldtag("InstanceFederationMode", "usage"))
+		cmd.Flags().Bool(InstanceFederationSpamFilterFlag(), cfg.InstanceFederationSpamFilter, fieldtag("InstanceFederationSpamFilter", "usage"))
 		cmd.Flags().Bool(InstanceExposePeersFlag(), cfg.InstanceExposePeers, fieldtag("InstanceExposePeers", "usage"))
 		cmd.Flags().Bool(InstanceExposeSuspendedFlag(), cfg.InstanceExposeSuspended, fieldtag("InstanceExposeSuspended", "usage"))
 		cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage"))

internal/config/helpers.gen.go

@@ -775,6 +775,31 @@ func GetInstanceFederationMode() string { return global.GetInstanceFederationMod
 // SetInstanceFederationMode safely sets the value for global configuration 'InstanceFederationMode' field
 func SetInstanceFederationMode(v string) { global.SetInstanceFederationMode(v) }
 
+// GetInstanceFederationSpamFilter safely fetches the Configuration value for state's 'InstanceFederationSpamFilter' field
+func (st *ConfigState) GetInstanceFederationSpamFilter() (v bool) {
+	st.mutex.RLock()
+	v = st.config.InstanceFederationSpamFilter
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceFederationSpamFilter safely sets the Configuration value for state's 'InstanceFederationSpamFilter' field
+func (st *ConfigState) SetInstanceFederationSpamFilter(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceFederationSpamFilter = v
+	st.reloadToViper()
+}
+
+// InstanceFederationSpamFilterFlag returns the flag name for the 'InstanceFederationSpamFilter' field
+func InstanceFederationSpamFilterFlag() string { return "instance-federation-spam-filter" }
+
+// GetInstanceFederationSpamFilter safely fetches the value for global configuration 'InstanceFederationSpamFilter' field
+func GetInstanceFederationSpamFilter() bool { return global.GetInstanceFederationSpamFilter() }
+
+// SetInstanceFederationSpamFilter safely sets the value for global configuration 'InstanceFederationSpamFilter' field
+func SetInstanceFederationSpamFilter(v bool) { global.SetInstanceFederationSpamFilter(v) }
+
 // GetInstanceExposePeers safely fetches the Configuration value for state's 'InstanceExposePeers' field
 func (st *ConfigState) GetInstanceExposePeers() (v bool) {
 	st.mutex.RLock()

internal/db/bundb/bundb_test.go

@@ -20,10 +20,10 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/federation/dereferencing/dereferencer_test.go

@@ -22,11 +22,11 @@
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/federation/federatingdb/create.go

@@ -21,13 +21,11 @@
 	"context"
 	"errors"
 	"fmt"
-	"strings"
 
 	"codeberg.org/gruf/go-logger/v2/level"
 	"github.com/miekg/dns"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -35,7 +33,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
-	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 // Create adds a new entry to the database which must be able to be
@@ -321,26 +318,45 @@ func (f *federatingDB) createStatusable(
 	statusable ap.Statusable,
 	forwarded bool,
 ) error {
-
+	// Check whether this status is both
-	// Check whether we should accept this new status,
+	// relevant, and doesn't look like spam.
-	// we do this BEFORE even handling forwards to us.
+	err := f.spamFilter.StatusableOK(ctx,
-	accept, err := f.shouldAcceptStatusable(ctx,
 		receiver,
 		requester,
 		statusable,
 	)
-	if err != nil {
-		return gtserror.Newf("error checking status acceptibility: %w", err)
-	}
 
-	if !accept {
+	switch {
-		// This is a status sent with no relation to receiver, i.e.
+	case err == nil:
-		// - receiving account does not follow requesting account
+		// No problem!
-		// - received status does not mention receiving account
+
+	case gtserror.IsNotRelevant(err):
+		// This case is quite common if a remote (Mastodon)
+		// instance forwards a message to us which is a reply
+		// from someone else to a status we've also replied to.
 		//
-		// We just pretend that all is fine (dog with cuppa, flames everywhere)
+		// It does this to try to ensure thread completion, but
-		log.Trace(ctx, "status failed acceptability check")
+		// we have our own thread fetching mechanism anyway.
+		log.Debugf(ctx,
+			"status %s is not relevant to receiver (%v); dropping it",
+			ap.GetJSONLDId(statusable), err,
+		)
 		return nil
+
+	case gtserror.IsSpam(err):
+		// Log this at a higher level so admins can
+		// gauge how much spam is being sent to them.
+		//
+		// TODO: add Prometheus metrics for this.
+		log.Infof(ctx,
+			"status %s looked like spam (%v); dropping it",
+			ap.GetJSONLDId(statusable), err,
+		)
+		return nil
+
+	default:
+		// A real error has occurred.
+		return gtserror.Newf("error checking relevancy/spam: %w", err)
 	}
 
 	// If we do have a forward, we should ignore the content
@@ -378,52 +394,6 @@ func (f *federatingDB) createStatusable(
 	return nil
 }
 
-func (f *federatingDB) shouldAcceptStatusable(ctx context.Context, receiver *gtsmodel.Account, requester *gtsmodel.Account, statusable ap.Statusable) (bool, error) {
-	host := config.GetHost()
-	accountDomain := config.GetAccountDomain()
-
-	// Check whether status mentions the receiver,
-	// this is the quickest check so perform it first.
-	mentions, _ := ap.ExtractMentions(statusable)
-	for _, mention := range mentions {
-
-		// Extract placeholder mention vars.
-		accURI := mention.TargetAccountURI
-		name := mention.NameString
-
-		switch {
-		case accURI != "" &&
-			accURI == receiver.URI || accURI == receiver.URL:
-			// Mention target is receiver,
-			// they are mentioned in status.
-			return true, nil
-
-		case accURI == "" && name != "":
-			// Only a name was provided, extract the user@domain parts.
-			user, domain, err := util.ExtractNamestringParts(name)
-			if err != nil {
-				return false, gtserror.Newf("error extracting mention name parts: %w", err)
-			}
-
-			// Check if the name points to our receiving local user.
-			isLocal := (domain == host || domain == accountDomain)
-			if isLocal && strings.EqualFold(user, receiver.Username) {
-				return true, nil
-			}
-		}
-	}
-
-	// Check whether receiving account follows the requesting account.
-	follows, err := f.state.DB.IsFollowing(ctx, receiver.ID, requester.ID)
-	if err != nil {
-		return false, gtserror.Newf("error checking follow status: %w", err)
-	}
-
-	// Status will only be acceptable
-	// if receiver follows requester.
-	return follows, nil
-}
-
 /*
 	FOLLOW HANDLERS
 */

internal/federation/federatingdb/db.go

@@ -22,12 +22,14 @@
 
 	"github.com/superseriousbusiness/activity/pub"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/spam"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
-// DB wraps the pub.Database interface with a couple of custom functions for GoToSocial.
+// DB wraps the pub.Database interface with
+// a couple of custom functions for GoToSocial.
 type DB interface {
 	pub.Database
 	Undo(ctx context.Context, undo vocab.ActivityStreamsUndo) error
@@ -36,20 +38,28 @@ type DB interface {
 	Announce(ctx context.Context, announce vocab.ActivityStreamsAnnounce) error
 }
 
-// FederatingDB uses the underlying DB interface to implement the go-fed pub.Database interface.
+// FederatingDB uses the given state interface
-// It doesn't care what the underlying implementation of the DB interface is, as long as it works.
+// to implement the go-fed pub.Database interface.
 type federatingDB struct {
 	state      *state.State
 	converter  *typeutils.Converter
-	filter    *visibility.Filter
+	visFilter  *visibility.Filter
+	spamFilter *spam.Filter
 }
 
-// New returns a DB interface using the given database and config
+// New returns a DB that satisfies the pub.Database
-func New(state *state.State, converter *typeutils.Converter, filter *visibility.Filter) DB {
+// interface, using the given state and filters.
+func New(
+	state *state.State,
+	converter *typeutils.Converter,
+	visFilter *visibility.Filter,
+	spamFilter *spam.Filter,
+) DB {
 	fdb := federatingDB{
 		state:      state,
 		converter:  converter,
-		filter:    filter,
+		visFilter:  visFilter,
+		spamFilter: spamFilter,
 	}
 	return &fdb
 }

internal/federation/federatingdb/federatingdb_test.go

@@ -23,12 +23,12 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/federation/federator_test.go

@@ -23,12 +23,12 @@
 	"github.com/stretchr/testify/suite"
 
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/filter/spam/spam.go

@@ -0,0 +1,32 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package spam
+
+import "github.com/superseriousbusiness/gotosocial/internal/state"
+
+// Filter packages logic for checking whether
+// given statuses should be considered spam.
+type Filter struct {
+	state *state.State
+}
+
+// NewFilter returns a new spam Filter
+// that will use the provided state.
+func NewFilter(state *state.State) *Filter {
+	return &Filter{state: state}
+}

internal/filter/spam/spam_test.go

@@ -0,0 +1,59 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package spam_test
+
+import (
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/spam"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type FilterStandardTestSuite struct {
+	// standard suite interfaces
+	suite.Suite
+	db    db.DB
+	state state.State
+
+	// standard suite models
+	testAccounts map[string]*gtsmodel.Account
+
+	filter *spam.Filter
+}
+
+func (suite *FilterStandardTestSuite) SetupSuite() {
+	suite.testAccounts = testrig.NewTestAccounts()
+}
+
+func (suite *FilterStandardTestSuite) SetupTest() {
+	suite.state.Caches.Init()
+
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+
+	suite.db = testrig.NewTestDB(&suite.state)
+	suite.filter = spam.NewFilter(&suite.state)
+
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *FilterStandardTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}

internal/filter/spam/statusable.go

@@ -0,0 +1,472 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package spam
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"slices"
+	"strings"
+
+	"github.com/miekg/dns"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/regexes"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// preppedMention represents a partially-parsed
+// mention, prepared for spam checking purposes.
+type preppedMention struct {
+	*gtsmodel.Mention
+	uri    *url.URL
+	domain string
+	user   string
+	local  bool
+}
+
+// StatusableOK returns no error if the given statusable looks OK,
+// ie., relevant to the receiver, and not spam.
+//
+// This should only be used for Creates of statusables, NOT Announces!
+//
+// If the statusable does not pass relevancy or spam checks, either
+// a Spam or NotRelevant error will be returned. Callers should use
+// gtserror.IsSpam() and gtserror.IsNotRelevant() to check for this.
+//
+// If the returned error is not nil, but neither Spam or NotRelevant,
+// then it's an actual database error.
+//
+// The decision is made based on the following heuristics, in order:
+//
+//  1. Receiver follow requester. Return nil.
+//  2. Statusable doesn't mention receiver. Return NotRelevant.
+//
+// If instance-federation-spam-filter = false, then return nil now.
+// Otherwise check:
+//
+//  3. Receiver is locked and is followed by requester. Return nil.
+//  4. Five or more people are mentioned. Return Spam.
+//  5. Receiver follow (requests) a mentioned account. Return nil.
+//  6. Statusable has a media attachment. Return Spam.
+//  7. Statusable contains non-mention, non-hashtag links. Return Spam.
+func (f *Filter) StatusableOK(
+	ctx context.Context,
+	receiver *gtsmodel.Account,
+	requester *gtsmodel.Account,
+	statusable ap.Statusable,
+) error {
+	// HEURISTIC 1: Check whether receiving account follows the requesting account.
+	// If so, we know it's OK and don't need to do any other checks.
+	follows, err := f.state.DB.IsFollowing(ctx, receiver.ID, requester.ID)
+	if err != nil {
+		return gtserror.Newf("db error checking follow status: %w", err)
+	}
+
+	if follows {
+		// Looks fine.
+		return nil
+	}
+
+	// HEURISTIC 2: Check whether statusable mentions the
+	// receiver. If not, we don't want to process this message.
+	rawMentions, _ := ap.ExtractMentions(statusable)
+	mentions := prepMentions(ctx, rawMentions)
+	mentioned := f.isMentioned(ctx, receiver, mentions)
+	if !mentioned {
+		// This is a random message fired
+		// into our inbox, just drop it.
+		err := errors.New("receiver does not follow requester, and is not mentioned")
+		return gtserror.SetNotRelevant(err)
+	}
+
+	// Receiver is mentioned, but not by someone
+	// they follow. Check if we need to do more
+	// granular spam filtering.
+	if !config.GetInstanceFederationSpamFilter() {
+		// Filter is not enabled, allow it
+		// through without further checks.
+		return nil
+	}
+
+	// More granular spam filtering time!
+	//
+	// HEURISTIC 3: Does requester follow locked receiver?
+	followedBy, err := f.lockedFollowedBy(ctx, receiver, requester)
+	if err != nil {
+		return gtserror.Newf("db error checking follow status: %w", err)
+	}
+
+	// If receiver is locked, and is followed
+	// by requester, this likely means they're
+	// interested in the message. Allow it.
+	if followedBy {
+		return nil
+	}
+
+	// HEURISTIC 4: How many people are mentioned?
+	// If it's 5 or more we can assume this is spam.
+	mentionsLen := len(mentions)
+	if mentionsLen >= 5 {
+		err := errors.New("status mentions 5 or more people")
+		return gtserror.SetSpam(err)
+	}
+
+	// HEURISTIC 5: Four or fewer people are mentioned,
+	// do we follow (request) at least one of them?
+	// If so, we're probably interested in the message.
+	knowsOne := f.knowsOneMentioned(ctx, receiver, mentions)
+	if knowsOne {
+		return nil
+	}
+
+	// HEURISTIC 6: Are there any media attachments?
+	attachments, _ := ap.ExtractAttachments(statusable)
+	hasAttachments := len(attachments) != 0
+	if hasAttachments {
+		err := errors.New("status has attachment(s)")
+		return gtserror.SetSpam(err)
+	}
+
+	// HEURISTIC 7: Are there any links in the post
+	// aside from mentions and hashtags? Include the
+	// summary/content warning when checking.
+	hashtags, _ := ap.ExtractHashtags(statusable)
+	hasErrantLinks := f.errantLinks(ctx, statusable, mentions, hashtags)
+	if hasErrantLinks {
+		err := errors.New("status has one or more non-mention, non-hashtag links")
+		return gtserror.SetSpam(err)
+	}
+
+	// Looks OK.
+	return nil
+}
+
+// prepMentions prepares a slice of mentions
+// for spam checking by parsing out the namestring
+// and targetAccountURI values, if present.
+func prepMentions(
+	ctx context.Context,
+	mentions []*gtsmodel.Mention,
+) []preppedMention {
+	var (
+		host          = config.GetHost()
+		accountDomain = config.GetAccountDomain()
+	)
+
+	parsedMentions := make([]preppedMention, 0, len(mentions))
+	for _, mention := range mentions {
+		// Start by just embedding
+		// the original mention.
+		parsedMention := preppedMention{
+			Mention: mention,
+		}
+
+		// Try to parse namestring if present.
+		if mention.NameString != "" {
+			user, domain, err := util.ExtractNamestringParts(mention.NameString)
+			if err != nil {
+				// Malformed mention,
+				// just log + ignore.
+				log.Debugf(ctx,
+					"malformed mention namestring: %v",
+					err,
+				)
+				continue
+			}
+
+			parsedMention.domain = domain
+			parsedMention.user = user
+		}
+
+		// Try to parse URI if present.
+		if mention.TargetAccountURI != "" {
+			targetURI, err := url.Parse(mention.TargetAccountURI)
+			if err != nil {
+				// Malformed mention,
+				// just log + ignore.
+				log.Debugf(ctx,
+					"malformed mention uri: %v",
+					err,
+				)
+				continue
+			}
+
+			parsedMention.uri = targetURI
+
+			// Set host from targetURI if
+			// it wasn't set by namestring.
+			if parsedMention.domain == "" {
+				parsedMention.domain = targetURI.Host
+			}
+		}
+
+		// It's a mention of a local account if the target host is us.
+		parsedMention.local = parsedMention.domain == host || parsedMention.domain == accountDomain
+
+		// Done with this one.
+		parsedMentions = append(parsedMentions, parsedMention)
+	}
+
+	return parsedMentions
+}
+
+// isMentioned returns true if the
+// receiver is targeted by at least
+// one of the given mentions.
+func (f *Filter) isMentioned(
+	ctx context.Context,
+	receiver *gtsmodel.Account,
+	mentions []preppedMention,
+) bool {
+	return slices.ContainsFunc(
+		mentions,
+		func(mention preppedMention) bool {
+			// Check if receiver mentioned by URI.
+			if accURI := mention.TargetAccountURI; accURI != "" &&
+				(accURI == receiver.URI || accURI == receiver.URL) {
+				return true
+			}
+
+			// Check if receiver mentioned by namestring.
+			if mention.local && strings.EqualFold(mention.user, receiver.Username) {
+				return true
+			}
+
+			// Mention doesn't
+			// target receiver.
+			return false
+		},
+	)
+}
+
+// lockedFollowedBy returns true
+// if receiver account is locked,
+// and requester follows receiver.
+func (f *Filter) lockedFollowedBy(
+	ctx context.Context,
+	receiver *gtsmodel.Account,
+	requester *gtsmodel.Account,
+) (bool, error) {
+	// If receiver is not locked,
+	// return early to avoid a db call.
+	if !*receiver.Locked {
+		return false, nil
+	}
+
+	return f.state.DB.IsFollowing(ctx, requester.ID, receiver.ID)
+}
+
+// knowsOneMentioned returns true if the
+// receiver follows or has follow requested
+// at least one of the mentioned accounts.
+func (f *Filter) knowsOneMentioned(
+	ctx context.Context,
+	receiver *gtsmodel.Account,
+	mentions []preppedMention,
+) bool {
+	return slices.ContainsFunc(
+		mentions,
+		func(mention preppedMention) bool {
+			var (
+				acc *gtsmodel.Account
+				err error
+			)
+
+			// Try to get target account without
+			// dereffing. After all, if they're not
+			// in our db we definitely don't know them.
+			if mention.TargetAccountURI != "" {
+				acc, err = f.state.DB.GetAccountByURI(
+					gtscontext.SetBarebones(ctx),
+					mention.TargetAccountURI,
+				)
+			} else if mention.user != "" {
+				acc, err = f.state.DB.GetAccountByUsernameDomain(
+					gtscontext.SetBarebones(ctx),
+					mention.user,
+					mention.domain,
+				)
+			}
+
+			if err != nil && !errors.Is(err, db.ErrNoEntries) {
+				// Proper error.
+				log.Errorf(ctx, "db error getting mentioned account: %v", err)
+				return false
+			}
+
+			if acc == nil {
+				// We don't know this nerd!
+				return false
+			}
+
+			if acc.ID == receiver.ID {
+				// This is us, doesn't count.
+				return false
+			}
+
+			follows, err := f.state.DB.IsFollowing(ctx, receiver.ID, acc.ID)
+			if err != nil {
+				// Proper error.
+				log.Errorf(ctx, "db error checking follow status: %v", err)
+				return false
+			}
+
+			if follows {
+				// We follow this nerd.
+				return true
+			}
+
+			// We don't follow this nerd, but
+			// have we requested to follow them?
+			followRequested, err := f.state.DB.IsFollowRequested(ctx, receiver.ID, acc.ID)
+			if err != nil {
+				// Proper error.
+				log.Errorf(ctx, "db error checking follow req status: %v", err)
+				return false
+			}
+
+			return followRequested
+		},
+	)
+}
+
+// errantLinks returns true if any http/https
+// link discovered in the statusable content + cw
+// is not either a mention link, or a hashtag link.
+func (f *Filter) errantLinks(
+	ctx context.Context,
+	statusable ap.Statusable,
+	mentions []preppedMention,
+	hashtags []*gtsmodel.Tag,
+) bool {
+	// Concatenate the cw with the
+	// content to check for links in both.
+	cw := ap.ExtractSummary(statusable)
+	content := ap.ExtractContent(statusable)
+	concat := cw + " " + content.Content
+
+	// Store link string alongside link
+	// URI to avoid stringifying twice.
+	type preppedLink struct {
+		*url.URL
+		str string
+	}
+
+	// Find + parse every http/https link in the status.
+	rawLinks := regexes.LinkScheme.FindAllString(concat, -1)
+	links := make([]preppedLink, 0, len(rawLinks))
+	for _, rawLink := range rawLinks {
+		linkURI, err := url.Parse(rawLink)
+		if err != nil {
+			log.Debugf(ctx,
+				"malformed link in status: %v",
+				err,
+			)
+			// Ignore bad links
+			// for spam checking.
+			continue
+		}
+
+		links = append(links, preppedLink{
+			URL: linkURI,
+			str: rawLink,
+		})
+	}
+
+	// For each link in the status, try to
+	// match it to a hashtag or a mention.
+	// If we can't, we have an errant link.
+	for _, link := range links {
+		hashtagLink := slices.ContainsFunc(
+			hashtags,
+			func(hashtag *gtsmodel.Tag) bool {
+				// If a link is to the href
+				// of a hashtag, it's fine.
+				return strings.EqualFold(
+					link.str,
+					hashtag.Href,
+				)
+			},
+		)
+
+		if hashtagLink {
+			// This link is accounted for.
+			// Move to the next one.
+			continue
+		}
+
+		mentionLink := slices.ContainsFunc(
+			mentions,
+			func(mention preppedMention) bool {
+				// If link is straight up to the URI
+				// of a mentioned account, it's fine.
+				if strings.EqualFold(
+					link.str,
+					mention.TargetAccountURI,
+				) {
+					return true
+				}
+
+				// Link might be to an account URL rather
+				// than URI. This is a bit trickier because
+				// we can't predict the format of such URLs,
+				// and it's difficult to reconstruct them
+				// while also taking account of different
+				// host + account-domain values.
+				//
+				// So, just check if this link is on the same
+				// host as the mentioned account, or at least
+				// shares a host with it.
+				if link.Host == mention.domain {
+					// Same host.
+					return true
+				}
+
+				// Shares a host if it has at least two
+				// components from the right in common.
+				common := dns.CompareDomainName(
+					link.Host,
+					mention.domain,
+				)
+				return common >= 2
+			},
+		)
+
+		if mentionLink {
+			// This link is accounted for.
+			// Move to the next one.
+			continue
+		}
+
+		// Not a hashtag link
+		// or a mention link,
+		// so it's errant.
+		return true
+	}
+
+	// All links OK, or
+	// no links found.
+	return false
+}

internal/filter/spam/statusable_test.go

@@ -0,0 +1,780 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package spam_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+type StatusableTestSuite struct {
+	FilterStandardTestSuite
+}
+
+const (
+	// Message that mentions 5 people (including receiver),
+	// and contains a errant link.
+	spam1 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.org/users/9gol6f8zff",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "https://another.misskey.instance.com/users/9eklgce5yk",
+    "http://localhost:8080/users/the_mighty_zork"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><a href=\"https://spammylink.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">spammylink.org/</span><span class=\"invisible\"></span></a></p><p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.org/@Nao_ya_ia22\" class=\"u-url mention\">@<span>Nao_ya_ia22</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.org/users/9gol6f8zff",
+      "name": "@Nao_ya_ia22@example.org"
+    },
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+      "name": "@mendako@another.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/the_mighty_zork",
+      "name": "@the_mighty_zork@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 4 people (including receiver),
+	// and contains a errant link.
+	spam2 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "https://another.misskey.instance.com/users/9eklgce5yk",
+    "http://localhost:8080/users/the_mighty_zork"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><a href=\"https://spammylink.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">spammylink.org/</span><span class=\"invisible\"></span></a></p><p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+      "name": "@mendako@another.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/the_mighty_zork",
+      "name": "@the_mighty_zork@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 4 people (including receiver),
+	// but contains no errant links.
+	spam3 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "https://another.misskey.instance.com/users/9eklgce5yk",
+    "http://localhost:8080/users/the_mighty_zork"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+      "name": "@mendako@another.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/the_mighty_zork",
+      "name": "@the_mighty_zork@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 4 people (including receiver),
+	// contains no errant links, but 1 attachment.
+	spam4 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "https://another.misskey.instance.com/users/9eklgce5yk",
+    "http://localhost:8080/users/the_mighty_zork"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+  "attachment": [
+    {
+      "blurhash": "LNJRdVM{00Rj%Mayt7j[4nWBofRj",
+      "mediaType": "image/jpeg",
+      "name": "",
+      "type": "Document",
+      "url": "http://fossbros-anonymous.io/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
+    }
+  ],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+      "name": "@mendako@another.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/the_mighty_zork",
+      "name": "@the_mighty_zork@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 4 people (including receiver),
+	// and contains a errant link, and receiver follows
+	// another mentioned account.
+	spam5 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "http://localhost:8080/users/admin",
+    "http://localhost:8080/users/the_mighty_zork"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><a href=\"https://spammylink.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">spammylink.org/</span><span class=\"invisible\"></span></a></p><p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\">@<span>admin</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/admin",
+      "name": "@admin@localhost:8080"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/the_mighty_zork",
+      "name": "@the_mighty_zork@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 3 people, contains a
+	// errant link, and receiver follows another
+	// mentioned account. However, receiver is not mentioned.
+	spam6 = `{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2024-02-24T07:06:14Z",
+  "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+  "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "http://fossbros-anonymous.io/users/foss_satan/followers",
+    "https://example.net/users/nityosan",
+    "https://a.misskey.instance.com/users/9c06ylkgsx",
+    "http://localhost:8080/users/admin"
+  ],
+  "sensitive": false,
+  "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+  "inReplyToAtomUri": null,
+  "content": "<p><a href=\"https://spammylink.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">spammylink.org/</span><span class=\"invisible\"></span></a></p><p><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@admin\" class=\"u-url mention\">@<span>admin</span></a></span></p>",
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://example.net/users/nityosan",
+      "name": "@nityosan@example.net"
+    },
+    {
+      "type": "Mention",
+      "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "name": "@FIzxive@a.misskey.instance.com"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:8080/users/admin",
+      "name": "@admin@localhost:8080"
+    }
+  ],
+  "replies": {
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+      "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "items": []
+    }
+  }
+}`
+
+	// Message that mentions 4 people (including receiver),
+	// and hash a hashtag, but contains no errant links.
+	spam7 = `{
+    "@context": [
+      "https://www.w3.org/ns/activitystreams",
+      {
+        "ostatus": "http://ostatus.org#",
+        "atomUri": "ostatus:atomUri",
+        "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+        "conversation": "ostatus:conversation",
+        "sensitive": "as:sensitive",
+        "toot": "http://joinmastodon.org/ns#",
+        "votersCount": "toot:votersCount"
+      }
+    ],
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+    "type": "Note",
+    "summary": null,
+    "inReplyTo": null,
+    "published": "2024-02-24T07:06:14Z",
+    "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+    "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+    "to": [
+      "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+      "http://fossbros-anonymous.io/users/foss_satan/followers",
+      "https://example.net/users/nityosan",
+      "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "https://another.misskey.instance.com/users/9eklgce5yk",
+      "http://localhost:8080/users/the_mighty_zork"
+    ],
+    "sensitive": false,
+    "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+    "inReplyToAtomUri": null,
+    "content": "<p><a href=\"https://fossbros-anonymous.io/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag\">#<span>gotosocial</span></a> smells<br/><br/><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+    "attachment": [],
+    "tag": [
+      {
+        "type": "Mention",
+        "href": "https://example.net/users/nityosan",
+        "name": "@nityosan@example.net"
+      },
+      {
+        "type": "Mention",
+        "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+        "name": "@FIzxive@a.misskey.instance.com"
+      },
+      {
+        "type": "Mention",
+        "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+        "name": "@mendako@another.misskey.instance.com"
+      },
+      {
+        "type": "Mention",
+        "href": "http://localhost:8080/users/the_mighty_zork",
+        "name": "@the_mighty_zork@localhost:8080"
+      },
+      {
+        "type": "Hashtag",
+        "href": "https://fossbros-anonymous.io/tags/gotosocial",
+        "name": "#gotosocial"
+      }  
+    ],
+    "replies": {
+      "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "type": "Collection",
+      "first": {
+        "type": "CollectionPage",
+        "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+        "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+        "items": []
+      }
+    }
+  }`
+
+	// Same as spam7, except message doesn't
+	// have a hashtag in the tags array.
+	spam8 = `{
+    "@context": [
+      "https://www.w3.org/ns/activitystreams",
+      {
+        "ostatus": "http://ostatus.org#",
+        "atomUri": "ostatus:atomUri",
+        "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+        "conversation": "ostatus:conversation",
+        "sensitive": "as:sensitive",
+        "toot": "http://joinmastodon.org/ns#",
+        "votersCount": "toot:votersCount"
+      }
+    ],
+    "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+    "type": "Note",
+    "summary": null,
+    "inReplyTo": null,
+    "published": "2024-02-24T07:06:14Z",
+    "url": "http://fossbros-anonymous.io/@foss_satan/111985188827079562",
+    "attributedTo": "http://fossbros-anonymous.io/users/foss_satan",
+    "to": [
+      "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+      "http://fossbros-anonymous.io/users/foss_satan/followers",
+      "https://example.net/users/nityosan",
+      "https://a.misskey.instance.com/users/9c06ylkgsx",
+      "https://another.misskey.instance.com/users/9eklgce5yk",
+      "http://localhost:8080/users/the_mighty_zork"
+    ],
+    "sensitive": false,
+    "atomUri": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562",
+    "inReplyToAtomUri": null,
+    "content": "<p><a href=\"https://fossbros-anonymous.io/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag\">#<span>gotosocial</span></a> smells<br/><br/><span class=\"h-card\" translate=\"no\"><a href=\"https://example.net/@nityosan\" class=\"u-url mention\">@<span>nityosan</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://a.misskey.instance.com/@FIzxive\" class=\"u-url mention\">@<span>FIzxive</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"https://another.misskey.instance.com/@mendako\" class=\"u-url mention\">@<span>mendako</span></a></span><br /><span class=\"h-card\" translate=\"no\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\">@<span>the_mighty_zork</span></a></span></p>",
+    "attachment": [],
+    "tag": [
+      {
+        "type": "Mention",
+        "href": "https://example.net/users/nityosan",
+        "name": "@nityosan@example.net"
+      },
+      {
+        "type": "Mention",
+        "href": "https://a.misskey.instance.com/users/9c06ylkgsx",
+        "name": "@FIzxive@a.misskey.instance.com"
+      },
+      {
+        "type": "Mention",
+        "href": "https://another.misskey.instance.com/users/9eklgce5yk",
+        "name": "@mendako@another.misskey.instance.com"
+      },
+      {
+        "type": "Mention",
+        "href": "http://localhost:8080/users/the_mighty_zork",
+        "name": "@the_mighty_zork@localhost:8080"
+      }
+    ],
+    "replies": {
+      "id": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+      "type": "Collection",
+      "first": {
+        "type": "CollectionPage",
+        "next": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies?only_other_accounts=true&page=true",
+        "partOf": "http://fossbros-anonymous.io/users/foss_satan/statuses/111985188827079562/replies",
+        "items": []
+      }
+    }
+  }`
+)
+
+func (suite *StatusableTestSuite) TestStatusableOK() {
+	var (
+		ctx       = context.Background()
+		receiver  = suite.testAccounts["local_account_1"]
+		requester = suite.testAccounts["remote_account_1"]
+	)
+
+	type testStruct struct {
+		message string
+		check   func(error)
+	}
+
+	for _, test := range []testStruct{
+		{
+			// SPAM: status mentions 5 or more people
+			message: spam1,
+			check: func(err error) {
+				suite.True(gtserror.IsSpam(err), "expected Spam, got %+v", err)
+			},
+		},
+		{
+			// SPAM: receiver doesn't know a mentioned account, and status has attachments or errant links
+			message: spam2,
+			check: func(err error) {
+				suite.True(gtserror.IsSpam(err), "expected Spam, got %+v", err)
+			},
+		},
+		{
+			// NOT SPAM: receiver doesn't know a mentioned account, but status has no attachments or errant links
+			message: spam3,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			// SPAM: receiver doesn't know a mentioned account, and status has attachments or errant links
+			message: spam4,
+			check: func(err error) {
+				suite.True(gtserror.IsSpam(err), "expected Spam, got %+v", err)
+			},
+		},
+		{
+			// NOT SPAM: receiver knows a mentioned account
+			message: spam5,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			// SPAM: receiver does not follow requester, and is not mentioned
+			message: spam6,
+			check: func(err error) {
+				suite.True(gtserror.IsNotRelevant(err), "expected NotRelevant, got %+v", err)
+			},
+		},
+		{
+			// NOT SPAM: receiver doesn't know a mentioned account, but status has no attachments or errant links
+			message: spam7,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			// SPAM: receiver doesn't know a mentioned account, and status has attachments or errant links
+			message: spam8,
+			check: func(err error) {
+				suite.True(gtserror.IsSpam(err), "expected Spam, got %+v", err)
+			},
+		},
+	} {
+		rc := io.NopCloser(bytes.NewReader([]byte(test.message)))
+
+		statusable, err := ap.ResolveStatusable(ctx, rc)
+		if err != nil {
+			suite.FailNow(err.Error())
+		}
+
+		err = suite.filter.StatusableOK(ctx, receiver, requester, statusable)
+		test.check(err)
+	}
+
+	// Put a follow in place from receiver to requester.
+	fID := id.NewULID()
+	if err := suite.state.DB.PutFollow(ctx, &gtsmodel.Follow{
+		ID:              fID,
+		URI:             "http://localhost:8080/users/the_mighty_zork/follows/" + fID,
+		AccountID:       receiver.ID,
+		TargetAccountID: requester.ID,
+	}); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Run all the tests again. They should all
+	// be OK since receiver now follows requester.
+	for _, test := range []testStruct{
+		{
+			message: spam1,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam2,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam3,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam4,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam5,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam6,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam7,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+		{
+			message: spam8,
+			check: func(err error) {
+				suite.NoError(err, "expected not spam, got %+v", err)
+			},
+		},
+	} {
+		rc := io.NopCloser(bytes.NewReader([]byte(test.message)))
+
+		statusable, err := ap.ResolveStatusable(ctx, rc)
+		if err != nil {
+			suite.FailNow(err.Error())
+		}
+
+		err = suite.filter.StatusableOK(ctx, receiver, requester, statusable)
+		test.check(err)
+	}
+}
+
+func TestStatusableTestSuite(t *testing.T) {
+	suite.Run(t, &StatusableTestSuite{})
+}

internal/filter/visibility/filter_test.go

@@ -20,9 +20,9 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/gtserror/error.go

@@ -37,6 +37,8 @@
 	wrongTypeKey
 	smtpKey
 	malformedKey
+	notRelevantKey
+	spamKey
 )
 
 // IsUnretrievable indicates that a call to retrieve a resource
@@ -127,3 +129,30 @@ func IsMalformed(err error) bool {
 func SetMalformed(err error) error {
 	return errors.WithValue(err, malformedKey, struct{}{})
 }
+
+// IsNotRelevant checks error for a stored "notRelevant" flag.
+// This error is used when determining whether or not to store
+// + process an incoming AP message.
+func IsNotRelevant(err error) bool {
+	_, ok := errors.Value(err, notRelevantKey).(struct{})
+	return ok
+}
+
+// SetNotRelevant will wrap the given error to store a "notRelevant" flag,
+// returning wrapped error. See IsNotRelevant() for example use-cases.
+func SetNotRelevant(err error) error {
+	return errors.WithValue(err, notRelevantKey, struct{}{})
+}
+
+// IsSpam checks error for a stored "spam" flag. This error is used when
+// determining whether or not to store + process an incoming AP message.
+func IsSpam(err error) bool {
+	_, ok := errors.Value(err, spamKey).(struct{})
+	return ok
+}
+
+// SetSpam will wrap the given error to store a "spam" flag,
+// returning wrapped error. See IsSpam() for example use-cases.
+func SetSpam(err error) error {
+	return errors.WithValue(err, spamKey, struct{}{})
+}

internal/gtsmodel/tag.go

@@ -27,4 +27,5 @@ type Tag struct {
 	Name      string    `bun:",unique,nullzero,notnull"`                                    // (lowercase) name of the tag without the hash prefix
 	Useable   *bool     `bun:",nullzero,notnull,default:true"`                              // Tag is useable on this instance.
 	Listable  *bool     `bun:",nullzero,notnull,default:true"`                              // Tagged statuses can be listed on this instance.
+	Href      string    `bun:"-"`                                                           // Href of the hashtag. Will only be set on freshly-extracted hashtags from remote AP messages. Not stored in the database.
 }

internal/media/media_test.go

@@ -20,13 +20,13 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/account/account.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -26,7 +27,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // Processor wraps functionality for updating, creating, and deleting accounts in response to API requests.

internal/processing/account/account_test.go

@@ -24,6 +24,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
@@ -35,7 +36,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/admin/admin_test.go

@@ -23,6 +23,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
@@ -33,7 +34,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/common/common.go

@@ -19,9 +19,9 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // Processor provides a processor with logic

internal/processing/fedi/fedi.go

@@ -19,10 +19,10 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 type Processor struct {

internal/processing/polls/poll_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/stretchr/testify/suite"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -32,7 +33,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/polls"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/processor.go

@@ -21,6 +21,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	mm "github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -42,7 +43,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // Processor groups together processing functions and

internal/processing/processor_test.go

@@ -25,6 +25,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -34,7 +35,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/stream"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/search/search.go

@@ -19,9 +19,9 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 type Processor struct {

internal/processing/status/status.go

@@ -19,13 +19,13 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/polls"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 type Processor struct {

internal/processing/status/status_test.go

@@ -21,6 +21,7 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -31,7 +32,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/processing/timeline/home.go

@@ -23,6 +23,7 @@
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -30,7 +31,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // HomeTimelineGrab returns a function that satisfies GrabFunction for home timelines.

internal/processing/timeline/list.go

@@ -23,6 +23,7 @@
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -30,7 +31,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // ListTimelineGrab returns a function that satisfies GrabFunction for list timelines.

internal/processing/timeline/timeline.go

@@ -18,9 +18,9 @@
 package timeline
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 type Processor struct {

internal/processing/workers/surface.go

@@ -19,10 +19,10 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/email"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/stream"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // surface wraps functions for 'surfacing' the result

internal/processing/workers/workers.go

@@ -20,12 +20,12 @@
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/account"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/stream"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/workers"
 )
 

internal/processing/workers/workers_test.go

@@ -25,6 +25,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -34,7 +35,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/stream"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/timeline/timeline_test.go

@@ -22,10 +22,10 @@
 	"sort"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/transport/transport_test.go

@@ -24,6 +24,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -31,7 +32,6 @@
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

internal/typeutils/converter_test.go

@@ -21,11 +21,11 @@
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 

mkdocs.yml

@@ -118,6 +118,7 @@ nav:
       - "admin/cli.md"
       - "admin/backup_and_restore.md"
       - "admin/media_caching.md"
+      - "admin/spam.md"
   - "Federation":
       - "federation/index.md"
       - "federation/glossary.md"

test/envparsing.sh

@@ -87,6 +87,7 @@ EXPECT=$(cat << "EOF"
     "instance-expose-suspended": true,
     "instance-expose-suspended-web": true,
     "instance-federation-mode": "allowlist",
+    "instance-federation-spam-filter": true,
     "instance-inject-mastodon-version": true,
     "instance-languages": [
         "nl",
@@ -208,6 +209,7 @@ GTS_INSTANCE_EXPOSE_SUSPENDED=true \
 GTS_INSTANCE_EXPOSE_SUSPENDED_WEB=true \
 GTS_INSTANCE_EXPOSE_PUBLIC_TIMELINE=true \
 GTS_INSTANCE_FEDERATION_MODE='allowlist' \
+GTS_INSTANCE_FEDERATION_SPAM_FILTER=true \
 GTS_INSTANCE_DELIVER_TO_SHARED_INBOXES=false \
 GTS_INSTANCE_INJECT_MASTODON_VERSION=true \
 GTS_INSTANCE_LANGUAGES="nl,en-gb" \

testrig/config.go

@@ -65,6 +65,7 @@ func InitTestConfig() {
 	WebAssetBaseDir:    "./web/assets/",
 
 	InstanceFederationMode:         config.InstanceFederationModeDefault,
+	InstanceFederationSpamFilter:   true,
 	InstanceExposePeers:            true,
 	InstanceExposeSuspended:        true,
 	InstanceExposeSuspendedWeb:     true,

testrig/federatingdb.go

@@ -19,12 +19,18 @@
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/spam"
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // NewTestFederatingDB returns a federating DB with the underlying db
 func NewTestFederatingDB(state *state.State) federatingdb.DB {
-	return federatingdb.New(state, typeutils.NewConverter(state), visibility.NewFilter(state))
+	return federatingdb.New(
+		state,
+		typeutils.NewConverter(state),
+		visibility.NewFilter(state),
+		spam.NewFilter(state),
+	)
 }

testrig/util.go

@@ -27,13 +27,13 @@
 	"os"
 	"time"
 
+	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
 	wprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/workers"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
-	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
 // Starts workers on the provided state using noop processing functions.