diff --git a/README.md b/README.md index 3f9f357e2..19e5a2ee2 100644 --- a/README.md +++ b/README.md @@ -225,10 +225,11 @@ Simply download the binary + assets (or Docker container), tweak your configurat ### Safety + security features -- Built-in, automatic support for secure HTTPS with [Let's Encrypt](https://letsencrypt.org/). -- Strict privacy enforcement for posts and strict blocking logic. -- Import and export allow lists and deny lists. Subscribe to community-created block lists (think Ad blocker, but for federation!) (feature still in progress). +- Strict privacy enforcement for posts, and strict blocking logic. +- [Choose the visibility of posts on the web view of your profile](https://docs.gotosocial.org/en/latest/user_guide/settings/#visibility-level-of-posts-to-show-on-your-profile). +- [Import, export](https://docs.gotosocial.org/en/latest/admin/settings/#importexport), and [subscribe](https://docs.gotosocial.org/en/latest/admin/domain_permission_subscriptions) to community-created domain allow and domain block lists. - HTTP signature authentication: GoToSocial requires [HTTP Signatures](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-12) when sending and receiving messages, to ensure that your messages can't be tampered with and your identity can't be forged. +- Built-in, automatic support for secure HTTPS with [Let's Encrypt](https://letsencrypt.org/). ### Various federation modes diff --git a/ROADMAP.md b/ROADMAP.md index a4894f4fe..3b03475a2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -71,7 +71,7 @@ These are provided in no specific order. - [x] **Filters v2** -- implement v2 of the filters API. - [x] **Mute accounts** -- mute accounts to prevent their posts showing up in your home timeline (optional: for limited period of time). - [x] **Non-replyable posts** -- design a non-replyable post path for GoToSocial based on https://github.com/mastodon/mastodon/issues/14762#issuecomment-1196889788; allow users to create non-replyable posts. -- [ ] **Block + allow list subscriptions** -- allow instance admins to subscribe their instance to plaintext domain block/allow lists (much of the work for this is already in place). +- [x] **Block + allow list subscriptions** -- allow instance admins to subscribe their instance to domain block/allow lists. - [x] **Direct conversation view** -- allow users to easily page through all direct-message conversations they're a part of. - [ ] **Oauth token management** -- create / view / invalidate OAuth tokens via the settings panel. - [ ] **Status EDIT support** -- edit statuses that you've created, without having to delete + redraft. Federate edits out properly. diff --git a/cmd/gotosocial/action/admin/account/account.go b/cmd/gotosocial/action/admin/account/account.go index 57d0d3805..7dfb6b1d4 100644 --- a/cmd/gotosocial/action/admin/account/account.go +++ b/cmd/gotosocial/action/admin/account/account.go @@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) { state.Caches.Init() state.Caches.Start() - // Set the state DB connection + // Only set state DB connection. + // Don't need Actions or Workers for this (yet). dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbConn: %w", err) diff --git a/cmd/gotosocial/action/admin/media/list.go b/cmd/gotosocial/action/admin/media/list.go index 547954d4c..a017539ed 100644 --- a/cmd/gotosocial/action/admin/media/list.go +++ b/cmd/gotosocial/action/admin/media/list.go @@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) { state.Caches.Init() state.Caches.Start() + // Only set state DB connection. + // Don't need Actions or Workers for this. dbService, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbservice: %w", err) diff --git a/cmd/gotosocial/action/admin/media/prune/common.go b/cmd/gotosocial/action/admin/media/prune/common.go index 5b42a6687..d73676f5b 100644 --- a/cmd/gotosocial/action/admin/media/prune/common.go +++ b/cmd/gotosocial/action/admin/media/prune/common.go @@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) { state.Caches.Start() // Scheduler is required for the - // claner, but no other workers + // cleaner, but no other workers // are needed for this CLI action. state.Workers.StartScheduler() + // Set state DB connection. + // Don't need Actions for this. dbService, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbservice: %w", err) diff --git a/cmd/gotosocial/action/admin/trans/export.go b/cmd/gotosocial/action/admin/trans/export.go index f76982a1b..dae2db7db 100644 --- a/cmd/gotosocial/action/admin/trans/export.go +++ b/cmd/gotosocial/action/admin/trans/export.go @@ -33,12 +33,12 @@ var Export action.GTSAction = func(ctx context.Context) error { var state state.State + // Only set state DB connection. + // Don't need Actions or Workers for this. dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return fmt.Errorf("error creating dbservice: %s", err) } - - // Set the state DB connection state.DB = dbConn exporter := trans.NewExporter(dbConn) diff --git a/cmd/gotosocial/action/admin/trans/import.go b/cmd/gotosocial/action/admin/trans/import.go index 1ebf587ff..d34c816bb 100644 --- a/cmd/gotosocial/action/admin/trans/import.go +++ b/cmd/gotosocial/action/admin/trans/import.go @@ -33,12 +33,12 @@ var Import action.GTSAction = func(ctx context.Context) error { var state state.State + // Only set state DB connection. + // Don't need Actions or Workers for this. dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return fmt.Errorf("error creating dbservice: %s", err) } - - // Set the state DB connection state.DB = dbConn importer := trans.NewImporter(dbConn) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 376ade13d..efedda9ec 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -32,6 +32,7 @@ "github.com/KimMachineGun/automemlimit/memlimit" "github.com/gin-gonic/gin" "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/cleaner" @@ -44,6 +45,7 @@ "github.com/superseriousbusiness/gotosocial/internal/metrics" "github.com/superseriousbusiness/gotosocial/internal/middleware" tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/tracing" "go.uber.org/automaxprocs/maxprocs" @@ -164,6 +166,10 @@ // Set DB on state. state.DB = dbService + // Set Actions on state, providing workers to + // Actions as well for triggering side effects. + state.AdminActions = admin.New(dbService, &state.Workers) + // Ensure necessary database instance prerequisites exist. if err := dbService.CreateInstanceAccount(ctx); err != nil { return fmt.Errorf("error creating instance account: %s", err) @@ -283,15 +289,18 @@ func(context.Context, time.Time) { // Create background cleaner. cleaner := cleaner.New(state) - // Now schedule background cleaning tasks. - if err := cleaner.ScheduleJobs(); err != nil { - return fmt.Errorf("error scheduling cleaner jobs: %w", err) - } + // Create subscriptions fetcher. + subscriptions := subscriptions.New( + state, + transportController, + typeConverter, + ) // Create the processor using all the // other services we've created so far. process = processing.NewProcessor( cleaner, + subscriptions, typeConverter, federator, oauthServer, @@ -302,6 +311,16 @@ func(context.Context, time.Time) { intFilter, ) + // Schedule background cleaning tasks. + if err := cleaner.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling cleaner jobs: %w", err) + } + + // Schedule background subscriptions updating. + if err := subscriptions.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling subscriptions jobs: %w", err) + } + // Initialize the specialized workers pools. state.Workers.Client.Init(messages.ClientMsgIndices()) state.Workers.Federator.Init(messages.FederatorMsgIndices()) diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 19588c70a..0036b7a7d 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -20,11 +20,9 @@ package testrig import ( - "bytes" "context" "errors" "fmt" - "io" "net/http" "os" "os/signal" @@ -47,6 +45,7 @@ "github.com/superseriousbusiness/gotosocial/internal/router" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/tracing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -159,16 +158,8 @@ testrig.StandardStorageSetup(state.Storage, "./testrig/media") // build backend handlers - transportController := testrig.NewTestTransportController(state, testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - r := io.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - Header: http.Header{ - "Content-Type": req.Header.Values("Accept"), - }, - }, nil - }, "")) + httpClient := testrig.NewMockHTTPClient(nil, "./testrig/media") + transportController := testrig.NewTestTransportController(state, httpClient) mediaManager := testrig.NewTestMediaManager(state) federator := testrig.NewTestFederator(state, transportController, mediaManager) @@ -314,11 +305,23 @@ // Create background cleaner. cleaner := cleaner.New(state) - // Now schedule background cleaning tasks. + // Schedule background cleaning tasks. if err := cleaner.ScheduleJobs(); err != nil { return fmt.Errorf("error scheduling cleaner jobs: %w", err) } + // Create subscriptions fetcher. + subscriptions := subscriptions.New( + state, + transportController, + typeConverter, + ) + + // Schedule background subscriptions updating. + if err := subscriptions.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling subscriptions jobs: %w", err) + } + // Finally start the main http server! if err := route.Start(); err != nil { return fmt.Errorf("error starting router: %w", err) diff --git a/docs/admin/domain_blocks.md b/docs/admin/domain_blocks.md index 76f11d8a5..f89c84a10 100644 --- a/docs/admin/domain_blocks.md +++ b/docs/admin/domain_blocks.md @@ -2,7 +2,7 @@ GoToSocial supports 'blocking'/'suspending' domains that you don't want your instance to federate with. In our documentation, the two terms 'block' and 'suspend' are used interchangeably with regard to domains, because they mean the same thing: preventing your instance and the instance running on the target domain from communicating with one another, effectively cutting off federation between the two instances. -You can view, create, and remove domain blocks and domain allows using the [instance admin panel](./settings.md#federation). +You can view, create, and remove domain blocks and domain allows using the [instance admin panel](./settings.md#domain-permissions). This document focuses on what domain blocks actually *do* and what side effects are processed when you create a new domain block. diff --git a/docs/admin/domain_permission_subscriptions.md b/docs/admin/domain_permission_subscriptions.md new file mode 100644 index 000000000..77ec831e1 --- /dev/null +++ b/docs/admin/domain_permission_subscriptions.md @@ -0,0 +1,145 @@ +# Domain Permission Subscriptions + +Via the [admin settings panel](./settings.md#subscriptions), you can create and manage domain permission subscriptions. + +Domain permission subscriptions allow you to specify a URL at which a permission list is hosted. Every 24hrs at 11pm (by default), your instance will fetch and parse each list you're subscribed to, in order of priority (highest to lowest), and create domain permissions (or domain permission drafts) based on entries discovered in the lists. + +Each domain permission subscription can be used to create domain allow or domain block entries. + +!!! warning + Currently, via blocklist subscriptions it is only possible to create "suspend" level domain blocks; other severities are not yet supported. Entries of severity "silence" or "limit" etc. on subscribed blocklists will be skipped. + +## Priority + +When you specify multiple domain permission subscriptions, they will be fetched and parsed in order of priority, from highest priority (255) to lowest priority (0). + +Permissions discovered on lists higher up in the priority ranking will override permissions on lists lower down in the priority ranking. + +For example, an instance admin subscribes to two allow lists, "Important List" at priority 255, and "Less Important List" at priority 128. Each of these subscribed lists contain an entry for `good-eggs.example.org`. + +The subscription with the higher priority is the one that now creates and manages the domain allow entry for `good-eggs.example.org`. + +If the subscription with the higher priority is removed, then the next time all the subscriptions are fetched, "Less Important List" will create (or take ownership of) the domain allow instead. + +## Orphan Permissions + +Domain permissions (blocks or allows) that are not currently managed by a domain permission subscription are considered "orphan" permissions. This includes permissions that an admin created in the settings panel by hand, or which were imported manually via the import/export page. + +If you wish, when creating a domain permission subscription, you can set ["adopt orphans"](./settings.md#adopt-orphan-permissions) to true for that subscription. If a domain permission subscription that is set to adopt orphans encounters an orphan permission which is *also present on the list at the subscription's URI*, then it will "adopt" the orphan by setting the orphan's subscription ID to its own ID. + +For example, an instance admin manually creates a domain block for the domain `horrid-trolls.example.org`. Later, they create a domain permission subscription for a block list that contains an entry for `horrid-trolls.example.org`, and they set "adopt orphans" to true. When their instance fetches and parses the list, and creates domain permission entries from it, then the orphan domain block for `horrid-trolls.example.org` gets adopted by the domain permission subscription. Now, if the domain permission subscription is removed, and the option to remove all permissions owned by the subscription is checked, then the domain block for `horrid-trolls.example.org` will also be removed. + +## Fun Stuff To Do With Domain Permission Subscriptions + +### 1. Create an allowlist-federation cluster. + +Domain permission subscriptions make it possible to easily create allowlist-federation clusters, ie., a group of instances can essentially form their own mini-fediverse, wherein each instance runs in [allowlist federation mode](./federation_modes.md#allowlist-federation-mode), and subscribes to a cooperatively-managed allowlist hosted somewhere. + +For example, instances `instance-a.example.org`, `instance-b.example.org`, and `instance-c.example.org` decide that they only want to federate with each other. + +Using some version management platform like GitHub, they host a plaintext-formatted allowlist at something like `https://raw.githubusercontent.com/our-cluster/allowlist/refs/heads/main/allows.txt`. + +The contents of the plaintext-formatted allowlist are as follows: + +```text +instance-a.example.org +instance-b.example.org +instance-c.example.org +``` + +Each instance admin sets their federation mode to `allowlist`, and creates a subscription to create allows from `https://raw.githubusercontent.com/our-cluster/allowlist/refs/heads/main/allows.txt`, which results in domain allow entries being created for their own domain, and for each other domain in the cluster. + +At some point, someone from `instance-d.example.org` asks (out of band) whether they can be added to the cluster. The existing admins agree, and update their plaintext-formatted allowlist to read: + +```text +instance-a.example.org +instance-b.example.org +instance-c.example.org +instance-d.example.org +``` + +The next time each instance fetches the list, a new domain allow entry will be created for `instance-d.example.org`, and it will be able to federate with the other domains on the list. + +### 2. Cooperatively manage a blocklist. + +Domain permission subscriptions make it easy to collaborate on and subscribe to shared blocklists of domains that host illegal / fashy / otherwise undesired accounts and content. + +For example, the admins of instances `instance-e.example.org`, `instance-f.example.org`, and `instance-g.example.org` decide that they are tired of duplicating work by playing whack-a-mole with bad actors. To make their lives easier, they decide to collaborate on a shared blocklist. + +Using some version management platform like GitHub, they host a blocklist at something like `https://raw.githubusercontent.com/baddies/blocklist/refs/heads/main/blocks.csv`. + +When someone discovers a new domain hosting an instance they don't like, they can open a pull request or similar against the list, to add the questionable instance to the domain. + +For example, someone gets an unpleasant reply from a new instance `fashy-arseholes.example.org`. Using their collaboration tools, they propose adding `fashy-arseholes.example.org` to the blocklist. After some deliberation and discussion, the domain is added to the list. + +The next time each of `instance-e.example.org`, `instance-f.example.org`, and `instance-g.example.org` fetch the block list, a block entry will be created for ``fashy-arseholes.example.org``. + +### 3. Subscribe to a blocklist, but ignore some of it. + +Say that `instance-g.example.org` in the previous section decides that they agree with most of the collaboratively-curated blocklist, but they actually would like to keep federating with ``fashy-arseholes.example.org`` for some godforsaken reason. + +This can be done in one of three ways: + +1. The admin of `instance-g.example.org` subscribes to the shared blocklist, but they do so with the ["create as drafts"](./settings.md#create-permissions-as-drafts) option set to true. When their instance fetches the blocklist, a draft block is created for `fashy-arseholes.example.org`. The admin of `instance-g` just leaves the permission as a draft, or rejects it, so it never comes into force. +2. Before the blocklist is re-fetched, the admin of `instance-g.example.org` creates a [domain permission exclude](./settings.md#excludes) entry for ``instance-g.example.org``. The domain ``instance-g.example.org`` then becomes exempt/excluded from automatic permission creation, and so the block for ``instance-g.example.org`` on the shared blocklist does not get created in the database of ``instance-g.example.org`` the next time the list is fetched. +3. The admin of `instance-g.example.org` creates an explicit domain allow entry for `fashy-arseholes.example.org` on their own instance. Because their instance is running in `blocklist` federation mode, [the explicit allow overrides the domain block entry](./federation_modes.md#in-blocklist-mode), and so the domain remains unblocked. + +### 4. Subscribe directly to another instance's blocklist. + +Because GoToSocial is able to fetch and parse JSON-formatted lists of domain permissions, it is possible to subscribe directly to another instance's list of blocked domains via their `/api/v1/instance/domain_blocks` (Mastodon) or `/api/v1/instance/peers?filter=suspended` (GoToSocial) endpoint (if exposed). + +For example, the Mastodon instance `peepee.poopoo.example.org` exposes their block list publicly, and the owner of the GoToSocial instance `instance-h.example.org` decides they quite like the cut of the Mastodon moderator's jib. They create a domain permission subscription of type JSON, and set the URI to `https://peepee.poopoo.example.org/api/v1/instance/domain_blocks`. Every 24 hours, their instance will go fetch the blocklist JSON from the Mastodon instance, and create permissions based on entries discovered therein. + +## Example lists per content type + +Shown below are examples of the different permission list formats that GoToSocial is able to understand and parse. + +Each list contains three domains, `bumfaces.net`, `peepee.poopoo`, and `nothanks.com`. + +### CSV + +CSV lists use content type `text/csv`. + +Mastodon domain permission exports generally use this format. + +```csv +#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate +bumfaces.net,suspend,false,false,big jerks,false +peepee.poopoo,suspend,false,false,harassment,false +nothanks.com,suspend,false,false,,false +``` + +### JSON (application/json) + +JSON lists use content type `application/json`. + +```json +[ + { + "domain": "bumfaces.net", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "big jerks" + }, + { + "domain": "peepee.poopoo", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "harassment" + }, + { + "domain": "nothanks.com", + "suspended_at": "2020-05-13T13:29:12.000Z" + } +] +``` + +### Plaintext (text/plain) + +Plaintext lists use content type `text/plain`. + +Note that it is not possible to include any fields like "obfuscate" or "public comment" in plaintext lists, as they are simply a newline-separated list of domains. + +```text +bumfaces.net +peepee.poopoo +nothanks.com +``` diff --git a/docs/admin/settings.md b/docs/admin/settings.md index 170d07e6a..37581e177 100644 --- a/docs/admin/settings.md +++ b/docs/admin/settings.md @@ -34,11 +34,11 @@ Clicking on the username of the reported account opens that account in the 'Acco You can use this section to search for an account and perform moderation actions on it. -### Federation +### Domain Permissions ![List of suspended instances, with a field to filter/add new blocks. Below is a link to the bulk import/export interface](../public/admin-settings-federation.png) -In the federation section you can create, delete, and review explicit domain blocks and domain allows. +In the domain permissions section you can create, delete, and review domain blocks, domain allows, drafts, excludes, and subscriptions. For more detail on federation settings, and specifically how domain allows and domain blocks work in combination, please see [the federation modes section](./federation_modes.md), and [the domain blocks section](./domain_blocks.md). @@ -46,20 +46,99 @@ For more detail on federation settings, and specifically how domain allows and d You can enter a domain to suspend in the search field, which will filter the list to show you if you already have a block for it. -Clicking 'suspend' gives you a form to add a public and/or private comment, and submit to add the block. Adding a suspension will suspend all the currently known accounts on the instance, and prevent any new interactions with any user on the blocked instance. +Clicking 'suspend' gives you a form to add a public and/or private comment, and submit to add the block. + +Adding a domain block will suspend all currently known accounts from that domain, and prevent any new interactions with the blocked domain. #### Domain Allows The domain allows section works much like the domain blocks section, described above, only for explicit domain allows rather than domain blocks. -#### Bulk import/export +#### Import/export -Through the link at the bottom of the Federation section (or going to `/settings/admin/federation/import-export`) you can do bulk import/export of blocklists and allowlists. +In this section you can do bulk import/export of domain permissions in JSON, CSV, or plaintext formats. ![List of domains included in an import, providing ways to select some or all of them, change their domains, and update the use of subdomains.](../public/admin-settings-federation-import-export.png) Upon importing a list, either through the input field or from a file, you can review the entries in the list before importing a subset. You'll also be warned for entries that use subdomains, providing an easy way to change them to the main domain. +#### Drafts + +In this section you can create, search through, accept, and reject domain permission drafts. + +Domain permission drafts are domain permissions that have been proposed (either via manual creation or as an entry from a subscribed block / allow list), but have not yet come into force. + +Until it is accepted, a domain permission draft will not have any effect on federation with the domain it targets. Upon acceptance, it will be converted into either a domain block or a domain allow, and start being enforced. + +#### Excludes + +In this section, you can create, search through, and remove domain permission excludes. + +Domain permission excludes prevent permissions for a domain (and all subdomains) from being automatically managed by domain permission subscriptions. + +For example, if you create an exclude entry for the domain `example.org`, then a blocklist or allowlist subscription will exclude entries for `example.org` and any of its subdomains (`sub.example.org`, `another.sub.example.org` etc.) when creating domain permission drafts and domain blocks/allows. + +This functionality allows you to manually manage permissions for excluded domains, in cases where you know you definitely do or don't want to federate with a given domain, no matter what entries are contained in a domain permission subscription. + +Note that by itself, creation of an exclude entry for a given domain does not affect federation with that domain at all, it is only useful in combination with permission subscriptions. + +#### Subscriptions + +In this section, you can create, search through, edit, test, and remove domain permission subscriptions. + +Domain permission subscriptions allow you to specify a URL at which a permission list is hosted. Every 24hrs at 11pm (by default), your instance will fetch and parse each subscribed list, and create domain permissions (or domain permission drafts) based on entries in the lists. + +##### Title + +You can optionally use the title field to set a title for the subscription, as a reminder for yourself and other admins. + +For example, you might subscribe to a list at `https://lists.example.org/baddies.csv` and set the title of the subscription to something that reflects the contents of that list, such as "Basic block list (worst of the worst)", or similar. + +##### Subscription Priority + +When you specify multiple domain permission subscriptions, they will be fetched and parsed in order of priority, from highest priority (255) to lowest priority (0). + +Permissions discovered on lists higher up in the priority ranking will override permissions on lists lower down in the priority ranking. + +For more information on priority, please see the separate [domain permission subscriptions](./domain_permission_subscriptions.md) document. + +##### Permission Type + +You can use this dropdown to select whether permissions discovered at the list URL should be created as domain blocks, or domain allows. + +##### Content Type + +You can use this dropdown to select the content type of the list at the subscribed URL. + +Use CSV for Mastodon-style permission lists, plain for plaintext lists of domain names, or JSON for json-exported lists. + +##### Basic Auth + +Check this box to provide a basic auth username and/or password credential for the subscribed list, which will be sent along with each request to fetch the list. + +##### Adopt Orphan Permissions + +If you check this box, then any existing domain permissions will become managed by this subscription in the following circumstances: + +1. They don't already have a subscription ID (ie., they're not managed by any domain permission subscription). +2. They match a domain permission included in the list at the URL of this subscription. + +For more information on orphan permissions, please see the separate [domain permission subscriptions](./domain_permission_subscriptions.md) document. + +##### Create Permissions as Drafts + +With this box checked (default), any permissions created by this subscription will be created as **drafts** which require manual approval to come into force. + +It is recommended to leave this box checked unless you absolutely trust the subscribed list, to avoid inadvertent blocking or allowing of domains you'd rather not block or allow. + +##### Test a Subscription + +To test whether a subscription can be successfully parsed, first create the subscription, then in the detailed view for that subscription, click on the "Test" button. + +If your instance is able to fetch and parse permissions at the subscription URI, then you will see a list of these after clicking "Test". Otherwise, you will see an error message. + +![Screenshot of the detailed view of a subscription, with arrows pointing to the test section near the bottom.](../public/admin-settings-federation-subscription-test.png) + ## Administration Instance administration settings. diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 235309ba9..322b79b16 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -6436,6 +6436,46 @@ paths: summary: Remove a domain permission subscription. tags: - admin + /api/v1/admin/domain_permission_subscriptions/{id}/test: + post: + description: |- + The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message. + + This is useful in cases where you want to check that your instance can actually fetch + parse a list. + operationId: domainPermissionSubscriptionTest + parameters: + - description: ID of the domain permission draft. + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: Either an array of domain permissions, OR an error message of the form `{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched. + schema: + items: + $ref: '#/definitions/domain' + type: array + "400": + description: bad request + "401": + description: unauthorized + "403": + description: forbidden + "406": + description: not acceptable + "409": + description: conflict + "500": + description: internal server error + security: + - OAuth2 Bearer: + - admin + summary: Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*. + tags: + - admin /api/v1/admin/domain_permission_subscriptions/preview: get: description: This view allows you to see the order in which domain permissions will actually be fetched and created. diff --git a/docs/overrides/public/admin-settings-federation-subscription-test.png b/docs/overrides/public/admin-settings-federation-subscription-test.png new file mode 100644 index 000000000..8ee9a6aa1 Binary files /dev/null and b/docs/overrides/public/admin-settings-federation-subscription-test.png differ diff --git a/docs/overrides/public/admin-settings-federation.png b/docs/overrides/public/admin-settings-federation.png index 215861b79..de321fdd2 100644 Binary files a/docs/overrides/public/admin-settings-federation.png and b/docs/overrides/public/admin-settings-federation.png differ diff --git a/internal/processing/admin/actions.go b/internal/admin/actions.go similarity index 75% rename from internal/processing/admin/actions.go rename to internal/admin/actions.go index 968e45baa..057bfe07d 100644 --- a/internal/processing/admin/actions.go +++ b/internal/admin/actions.go @@ -23,11 +23,12 @@ "sync" "time" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/workers" ) func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode { @@ -42,15 +43,34 @@ func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode { } type Actions struct { - r map[string]*gtsmodel.AdminAction - state *state.State + // Map of running actions. + running map[string]*gtsmodel.AdminAction - // Not embedded struct, - // to shield from access - // by outside packages. + // Lock for running admin actions. + // + // Not embedded struct, to shield + // from access by outside packages. m sync.Mutex + + // DB for storing, updating, + // deleting admin actions etc. + db db.DB + + // Workers for queuing + // admin action side effects. + workers *workers.Workers } +func New(db db.DB, workers *workers.Workers) *Actions { + return &Actions{ + running: make(map[string]*gtsmodel.AdminAction), + db: db, + workers: workers, + } +} + +type ActionF func(context.Context) gtserror.MultiError + // Run runs the given admin action by executing the supplied function. // // Run handles locking, action insertion and updating, so you don't have to! @@ -62,10 +82,10 @@ type Actions struct { // will be updated on the provided admin action in the database. func (a *Actions) Run( ctx context.Context, - action *gtsmodel.AdminAction, - f func(context.Context) gtserror.MultiError, + adminAction *gtsmodel.AdminAction, + f ActionF, ) gtserror.WithCode { - actionKey := action.Key() + actionKey := adminAction.Key() // LOCK THE MAP HERE, since we're // going to do some operations on it. @@ -73,7 +93,7 @@ func (a *Actions) Run( // Bail if an action with // this key is already running. - running, ok := a.r[actionKey] + running, ok := a.running[actionKey] if ok { a.m.Unlock() return errActionConflict(running) @@ -81,7 +101,7 @@ func (a *Actions) Run( // Action with this key not // yet running, create it. - if err := a.state.DB.PutAdminAction(ctx, action); err != nil { + if err := a.db.PutAdminAction(ctx, adminAction); err != nil { err = gtserror.Newf("db error putting admin action %s: %w", actionKey, err) // Don't store in map @@ -92,7 +112,7 @@ func (a *Actions) Run( // Action was inserted, // store in map. - a.r[actionKey] = action + a.running[actionKey] = adminAction // UNLOCK THE MAP HERE, since // we're done modifying it for now. @@ -104,22 +124,22 @@ func (a *Actions) Run( // Run the thing and collect errors. if errs := f(ctx); errs != nil { - action.Errors = make([]string, 0, len(errs)) + adminAction.Errors = make([]string, 0, len(errs)) for _, err := range errs { - action.Errors = append(action.Errors, err.Error()) + adminAction.Errors = append(adminAction.Errors, err.Error()) } } // Action is no longer running: // remove from running map. a.m.Lock() - delete(a.r, actionKey) + delete(a.running, actionKey) a.m.Unlock() // Mark as completed in the db, // storing errors for later review. - action.CompletedAt = time.Now() - if err := a.state.DB.UpdateAdminAction(ctx, action, "completed_at", "errors"); err != nil { + adminAction.CompletedAt = time.Now() + if err := a.db.UpdateAdminAction(ctx, adminAction, "completed_at", "errors"); err != nil { log.Errorf(ctx, "db error marking action %s as completed: %q", actionKey, err) } }() @@ -135,8 +155,8 @@ func (a *Actions) GetRunning() []*gtsmodel.AdminAction { defer a.m.Unlock() // Assemble all currently running actions. - running := make([]*gtsmodel.AdminAction, 0, len(a.r)) - for _, action := range a.r { + running := make([]*gtsmodel.AdminAction, 0, len(a.running)) + for _, action := range a.running { running = append(running, action) } @@ -166,5 +186,5 @@ func (a *Actions) TotalRunning() int { a.m.Lock() defer a.m.Unlock() - return len(a.r) + return len(a.running) } diff --git a/internal/processing/admin/actions_test.go b/internal/admin/actions_test.go similarity index 80% rename from internal/processing/admin/actions_test.go rename to internal/admin/actions_test.go index 9d12ae84d..c5084d955 100644 --- a/internal/processing/admin/actions_test.go +++ b/internal/admin/actions_test.go @@ -32,12 +32,26 @@ "github.com/superseriousbusiness/gotosocial/testrig" ) +const ( + rMediaPath = "../../testrig/media" + rTemplatePath = "../../web/template" +) + type ActionsTestSuite struct { - AdminStandardTestSuite + suite.Suite +} + +func (suite *ActionsTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() } func (suite *ActionsTestSuite) TestActionOverlap() { - ctx := context.Background() + var ( + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + ctx = context.Background() + ) + defer testrig.TearDownTestStructs(testStructs) // Suspend account. action1 := >smodel.AdminAction{ @@ -61,7 +75,7 @@ func (suite *ActionsTestSuite) TestActionOverlap() { key2 := action2.Key() suite.Equal("account/01H90S1CXQ97J9625C5YBXZWGT", key2) - errWithCode := suite.adminProcessor.Actions().Run( + errWithCode := testStructs.State.AdminActions.Run( ctx, action1, func(ctx context.Context) gtserror.MultiError { @@ -74,7 +88,7 @@ func(ctx context.Context) gtserror.MultiError { // While first action is sleeping, try to // process another with the same key. - errWithCode = suite.adminProcessor.Actions().Run( + errWithCode = testStructs.State.AdminActions.Run( ctx, action2, func(ctx context.Context) gtserror.MultiError { @@ -90,13 +104,13 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.AdminActions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } // Try again. - errWithCode = suite.adminProcessor.Actions().Run( + errWithCode = testStructs.State.AdminActions.Run( ctx, action2, func(ctx context.Context) gtserror.MultiError { @@ -107,14 +121,18 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.AdminActions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } } func (suite *ActionsTestSuite) TestActionWithErrors() { - ctx := context.Background() + var ( + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + ctx = context.Background() + ) + defer testrig.TearDownTestStructs(testStructs) // Suspend a domain. action := >smodel.AdminAction{ @@ -125,7 +143,7 @@ func (suite *ActionsTestSuite) TestActionWithErrors() { AccountID: "01H90S1ZZXP4N74H4A9RVW1MRP", } - errWithCode := suite.adminProcessor.Actions().Run( + errWithCode := testStructs.State.AdminActions.Run( ctx, action, func(ctx context.Context) gtserror.MultiError { @@ -140,13 +158,13 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.AdminActions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } // Get action from the db. - dbAction, err := suite.db.GetAdminAction(ctx, action.ID) + dbAction, err := testStructs.State.DB.GetAdminAction(ctx, action.ID) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/admin/domainkeys.go b/internal/admin/domainkeys.go new file mode 100644 index 000000000..ad943ce80 --- /dev/null +++ b/internal/admin/domainkeys.go @@ -0,0 +1,51 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "context" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (a *Actions) DomainKeysExpireF(domain string) ActionF { + return func(ctx context.Context) gtserror.MultiError { + var ( + expiresAt = time.Now() + errs gtserror.MultiError + ) + + // For each account on this domain, expire + // the public key and update the account. + if err := a.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) { + account.PublicKeyExpiresAt = expiresAt + if err := a.db.UpdateAccount(ctx, + account, + "public_key_expires_at", + ); err != nil { + errs.Appendf("db error updating account: %w", err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs + } +} diff --git a/internal/admin/domainperms.go b/internal/admin/domainperms.go new file mode 100644 index 000000000..fa686df43 --- /dev/null +++ b/internal/admin/domainperms.go @@ -0,0 +1,387 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "context" + "errors" + "time" + + "codeberg.org/gruf/go-kv" + + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/messages" +) + +// Returns an AdminActionF for +// domain allow side effects. +func (a *Actions) DomainAllowF( + actionID string, + domainAllow *gtsmodel.DomainAllow, +) ActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "allow"}, + {"actionID", actionID}, + {"domain", domainAllow.Domain}, + }...) + + // Log start + finish. + l.Info("processing side effects") + errs := a.domainAllowSideEffects(ctx, domainAllow) + l.Info("finished processing side effects") + + return errs + } +} + +func (a *Actions) domainAllowSideEffects( + ctx context.Context, + allow *gtsmodel.DomainAllow, +) gtserror.MultiError { + if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { + // We're running in allowlist mode, + // so there are no side effects to + // process here. + return nil + } + + // We're running in blocklist mode or + // some similar mode which necessitates + // domain allow side effects if a block + // was in place when the allow was created. + // + // So, check if there's a block. + block, err := a.db.GetDomainBlock(ctx, allow.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs := gtserror.NewMultiError(1) + errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) + return errs + } + + if block == nil { + // No block? + // No problem! + return nil + } + + // There was a block, over which the new + // allow ought to take precedence. To account + // for this, just run side effects as though + // the domain was being unblocked, while + // leaving the existing block in place. + // + // Any accounts that were suspended by + // the block will be unsuspended and be + // able to interact with the instance again. + return a.domainUnblockSideEffects(ctx, block) +} + +// Returns an AdminActionF for +// domain unallow side effects. +func (a *Actions) DomainUnallowF( + actionID string, + domainAllow *gtsmodel.DomainAllow, +) ActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "unallow"}, + {"actionID", actionID}, + {"domain", domainAllow.Domain}, + }...) + + // Log start + finish. + l.Info("processing side effects") + errs := a.domainUnallowSideEffects(ctx, domainAllow) + l.Info("finished processing side effects") + + return errs + } +} + +func (a *Actions) domainUnallowSideEffects( + ctx context.Context, + allow *gtsmodel.DomainAllow, +) gtserror.MultiError { + if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { + // We're running in allowlist mode, + // so there are no side effects to + // process here. + return nil + } + + // We're running in blocklist mode or + // some similar mode which necessitates + // domain allow side effects if a block + // was in place when the allow was removed. + // + // So, check if there's a block. + block, err := a.db.GetDomainBlock(ctx, allow.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs := gtserror.NewMultiError(1) + errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) + return errs + } + + if block == nil { + // No block? + // No problem! + return nil + } + + // There was a block, over which the previous + // allow was taking precedence. Now that the + // allow has been removed, we should put the + // side effects of the block back in place. + // + // To do this, process the block side effects + // again as though the block were freshly + // created. This will mark all accounts from + // the blocked domain as suspended, and clean + // up their follows/following, media, etc. + return a.domainBlockSideEffects(ctx, block) +} + +func (a *Actions) DomainBlockF( + actionID string, + domainBlock *gtsmodel.DomainBlock, +) ActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "block"}, + {"actionID", actionID}, + {"domain", domainBlock.Domain}, + }...) + + skip, err := a.skipBlockSideEffects(ctx, domainBlock.Domain) + if err != nil { + return err + } + + if skip != "" { + l.Infof("skipping side effects: %s", skip) + return nil + } + + l.Info("processing side effects") + errs := a.domainBlockSideEffects(ctx, domainBlock) + l.Info("finished processing side effects") + + return errs + } +} + +// domainBlockSideEffects processes the side effects of a domain block: +// +// 1. Strip most info away from the instance entry for the domain. +// 2. Pass each account from the domain to the processor for deletion. +// +// It should be called asynchronously, since it can take a while when +// there are many accounts present on the given domain. +func (a *Actions) domainBlockSideEffects( + ctx context.Context, + block *gtsmodel.DomainBlock, +) gtserror.MultiError { + var errs gtserror.MultiError + + // If we have an instance entry for this domain, + // update it with the new block ID and clear all fields + instance, err := a.db.GetInstance(ctx, block.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("db error getting instance %s: %w", block.Domain, err) + return errs + } + + if instance != nil { + // We had an entry for this domain. + columns := stubbifyInstance(instance, block.ID) + if err := a.db.UpdateInstance(ctx, instance, columns...); err != nil { + errs.Appendf("db error updating instance: %w", err) + return errs + } + } + + // For each account that belongs to this domain, + // process an account delete message to remove + // that account's posts, media, etc. + if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { + if err := a.workers.Client.Process(ctx, &messages.FromClientAPI{ + APObjectType: ap.ActorPerson, + APActivityType: ap.ActivityDelete, + GTSModel: block, + Origin: account, + Target: account, + }); err != nil { + errs.Append(err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs +} + +func (a *Actions) DomainUnblockF( + actionID string, + domainBlock *gtsmodel.DomainBlock, +) ActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "unblock"}, + {"actionID", actionID}, + {"domain", domainBlock.Domain}, + }...) + + l.Info("processing side effects") + errs := a.domainUnblockSideEffects(ctx, domainBlock) + l.Info("finished processing side effects") + + return errs + } +} + +// domainUnblockSideEffects processes the side effects of undoing a +// domain block: +// +// 1. Mark instance entry as no longer suspended. +// 2. Mark each account from the domain as no longer suspended, if the +// suspension origin corresponds to the ID of the provided domain block. +// +// It should be called asynchronously, since it can take a while when +// there are many accounts present on the given domain. +func (a *Actions) domainUnblockSideEffects( + ctx context.Context, + block *gtsmodel.DomainBlock, +) gtserror.MultiError { + var errs gtserror.MultiError + + // Update instance entry for this domain, if we have it. + instance, err := a.db.GetInstance(ctx, block.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("db error getting instance %s: %w", block.Domain, err) + } + + if instance != nil { + // We had an entry, update it to signal + // that it's no longer suspended. + instance.SuspendedAt = time.Time{} + instance.DomainBlockID = "" + if err := a.db.UpdateInstance( + ctx, + instance, + "suspended_at", + "domain_block_id", + ); err != nil { + errs.Appendf("db error updating instance: %w", err) + return errs + } + } + + // Unsuspend all accounts whose suspension origin was this domain block. + if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { + if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() { + // Account wasn't suspended, nothing to do. + return + } + + if account.SuspensionOrigin != block.ID { + // Account was suspended, but not by + // this domain block, leave it alone. + return + } + + // Account was suspended by this domain + // block, mark it as unsuspended. + account.SuspendedAt = time.Time{} + account.SuspensionOrigin = "" + + if err := a.db.UpdateAccount( + ctx, + account, + "suspended_at", + "suspension_origin", + ); err != nil { + errs.Appendf("db error updating account %s: %w", account.Username, err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs +} + +// skipBlockSideEffects checks if side effects of block creation +// should be skipped for the given domain, taking account of +// instance federation mode, and existence of any allows +// which ought to "shield" this domain from being blocked. +// +// If the caller should skip, the returned string will be non-zero +// and will be set to a reason why side effects should be skipped. +// +// - blocklist mode + allow exists: "..." (skip) +// - blocklist mode + no allow: "" (don't skip) +// - allowlist mode + allow exists: "" (don't skip) +// - allowlist mode + no allow: "" (don't skip) +func (a *Actions) skipBlockSideEffects( + ctx context.Context, + domain string, +) (string, gtserror.MultiError) { + var ( + skip string // Assume "" (don't skip). + errs gtserror.MultiError + ) + + // Never skip block side effects in allowlist mode. + fediMode := config.GetInstanceFederationMode() + if fediMode == config.InstanceFederationModeAllowlist { + return skip, errs + } + + // We know we're in blocklist mode. + // + // We want to skip domain block side + // effects if an allow is already + // in place which overrides the block. + + // Check if an explicit allow exists for this domain. + domainAllow, err := a.db.GetDomainAllow(ctx, domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("error getting domain allow: %w", err) + return skip, errs + } + + if domainAllow != nil { + skip = "running in blocklist mode, and an explicit allow exists for this domain" + return skip, errs + } + + return skip, errs +} diff --git a/internal/admin/util.go b/internal/admin/util.go new file mode 100644 index 000000000..27e81fbb3 --- /dev/null +++ b/internal/admin/util.go @@ -0,0 +1,99 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "context" + "errors" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// stubbifyInstance renders the given instance as a stub, +// removing most information from it and marking it as +// suspended. +// +// For caller's convenience, this function returns the db +// names of all columns that are updated by it. +func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string { + instance.Title = "" + instance.SuspendedAt = time.Now() + instance.DomainBlockID = domainBlockID + instance.ShortDescription = "" + instance.Description = "" + instance.Terms = "" + instance.ContactEmail = "" + instance.ContactAccountUsername = "" + instance.ContactAccountID = "" + instance.Version = "" + + return []string{ + "title", + "suspended_at", + "domain_block_id", + "short_description", + "description", + "terms", + "contact_email", + "contact_account_username", + "contact_account_id", + "version", + } +} + +// rangeDomainAccounts iterates through all accounts +// originating from the given domain, and calls the +// provided range function on each account. +// +// If an error is returned while selecting accounts, +// the loop will stop and return the error. +func (a *Actions) rangeDomainAccounts( + ctx context.Context, + domain string, + rangeF func(*gtsmodel.Account), +) error { + var ( + limit = 50 // Limit selection to avoid spiking mem/cpu. + maxID string // Start with empty string to select from top. + ) + + for { + // Get (next) page of accounts. + accounts, err := a.db.GetInstanceAccounts(ctx, domain, maxID, limit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + return gtserror.Newf("db error getting instance accounts: %w", err) + } + + if len(accounts) == 0 { + // No accounts left, we're done. + return nil + } + + // Set next max ID for paging down. + maxID = accounts[len(accounts)-1].ID + + // Call provided range function. + for _, account := range accounts { + rangeF(account) + } + } +} diff --git a/internal/api/activitypub/emoji/emojiget_test.go b/internal/api/activitypub/emoji/emojiget_test.go index 4d687a049..0eb1f5931 100644 --- a/internal/api/activitypub/emoji/emojiget_test.go +++ b/internal/api/activitypub/emoji/emojiget_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -73,6 +74,7 @@ func (suite *EmojiGetTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage suite.tc = typeutils.NewConverter(&suite.state) diff --git a/internal/api/activitypub/users/user_test.go b/internal/api/activitypub/users/user_test.go index 4d55aad3d..10eb33937 100644 --- a/internal/api/activitypub/users/user_test.go +++ b/internal/api/activitypub/users/user_test.go @@ -20,6 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -84,6 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/api/auth/auth_test.go b/internal/api/auth/auth_test.go index d77b1a3d4..c5ceba387 100644 --- a/internal/api/auth/auth_test.go +++ b/internal/api/auth/auth_test.go @@ -26,6 +26,7 @@ "github.com/gin-contrib/sessions/memstore" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/auth" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -84,6 +85,7 @@ func (suite *AuthStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage suite.mediaManager = testrig.NewTestMediaManager(&suite.state) diff --git a/internal/api/client/accounts/account_test.go b/internal/api/client/accounts/account_test.go index 2f8664756..db212af22 100644 --- a/internal/api/client/accounts/account_test.go +++ b/internal/api/client/accounts/account_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/accounts" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -85,6 +86,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index 68a088b4d..a5a16f35f 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -46,6 +46,7 @@ DomainPermissionSubscriptionsPathWithID = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview" DomainPermissionSubscriptionRemovePath = DomainPermissionSubscriptionsPathWithID + "/remove" + DomainPermissionSubscriptionTestPath = DomainPermissionSubscriptionsPathWithID + "/test" DomainKeysExpirePath = BasePath + "/domain_keys_expire" HeaderAllowsPath = BasePath + "/header_allows" HeaderAllowsPathWithID = HeaderAllowsPath + "/:" + apiutil.IDKey @@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler) attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler) attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler) + attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler) // header filtering administration routes attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go index 962ec3872..479f16f45 100644 --- a/internal/api/client/admin/admin_test.go +++ b/internal/api/client/admin/admin_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + adminactions "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = adminactions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/admin/domainpermissionsubscriptiontest.go b/internal/api/client/admin/domainpermissionsubscriptiontest.go new file mode 100644 index 000000000..395a1a69c --- /dev/null +++ b/internal/api/client/admin/domainpermissionsubscriptiontest.go @@ -0,0 +1,118 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest +// +// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*. +// +// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message. +// +// This is useful in cases where you want to check that your instance can actually fetch + parse a list. +// +// --- +// tags: +// - admin +// +// produces: +// - application/json +// +// parameters: +// - +// name: id +// required: true +// in: path +// description: ID of the domain permission draft. +// type: string +// +// security: +// - OAuth2 Bearer: +// - admin +// +// responses: +// '200': +// description: >- +// Either an array of domain permissions, OR an error message of the form +// `{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched. +// schema: +// type: array +// items: +// "$ref": "#/definitions/domain" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '406': +// description: not acceptable +// '409': +// description: conflict +// '500': +// description: internal server error +func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if !*authed.User.Admin { + err := fmt.Errorf("user %s not an admin", authed.User.ID) + apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if authed.Account.IsMoving() { + apiutil.ForbiddenAfterMove(c) + return + } + + if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) + return + } + + id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest( + c.Request.Context(), + authed.Account, + id, + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + apiutil.JSON(c, http.StatusOK, resp) +} diff --git a/internal/api/client/admin/domainpermissionsubscruptiontest_test.go b/internal/api/client/admin/domainpermissionsubscruptiontest_test.go new file mode 100644 index 000000000..46861aba1 --- /dev/null +++ b/internal/api/client/admin/domainpermissionsubscruptiontest_test.go @@ -0,0 +1,125 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type DomainPermissionSubscriptionTestTestSuite struct { + AdminStandardTestSuite +} + +func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() { + var ( + ctx = context.Background() + testAccount = suite.testAccounts["admin_account"] + permSub = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + + // Create a subscription for a CSV list of baddies. + err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub) + if err != nil { + suite.FailNow(err.Error()) + } + + // Prepare the request to the /test endpoint. + subPath := strings.ReplaceAll( + admin.DomainPermissionSubscriptionTestPath, + ":id", permSub.ID, + ) + path := "/api" + subPath + recorder := httptest.NewRecorder() + ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json") + ginCtx.Params = gin.Params{ + gin.Param{ + Key: apiutil.IDKey, + Value: permSub.ID, + }, + } + + // Trigger the handler. + suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx) + suite.Equal(http.StatusOK, recorder.Code) + + // Read the body back. + b, err := io.ReadAll(recorder.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := new(bytes.Buffer) + if err := json.Indent(dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + // Ensure expected. + suite.Equal(`[ + { + "domain": "bumfaces.net", + "public_comment": "big jerks" + }, + { + "domain": "peepee.poopoo", + "public_comment": "harassment" + }, + { + "domain": "nothanks.com" + } +]`, dst.String()) + + // No permissions should be created + // since this is a dry run / test. + blocked, err := suite.state.DB.AreDomainsBlocked( + ctx, + []string{"bumfaces.net", "peepee.poopoo", "nothanks.com"}, + ) + if err != nil { + suite.FailNow(err.Error()) + } + suite.False(blocked) +} + +func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) { + suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{}) +} diff --git a/internal/api/client/bookmarks/bookmarks_test.go b/internal/api/client/bookmarks/bookmarks_test.go index cb796e9e8..43c1eeee4 100644 --- a/internal/api/client/bookmarks/bookmarks_test.go +++ b/internal/api/client/bookmarks/bookmarks_test.go @@ -28,6 +28,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/bookmarks" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" @@ -95,6 +96,7 @@ func (suite *BookmarkTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/favourites/favourites_test.go b/internal/api/client/favourites/favourites_test.go index bd0ebce2e..dcdc8fee2 100644 --- a/internal/api/client/favourites/favourites_test.go +++ b/internal/api/client/favourites/favourites_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/favourites" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -79,6 +80,7 @@ func (suite *FavouritesStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/filters/v1/filter_test.go b/internal/api/client/filters/v1/filter_test.go index 7553008d3..128426435 100644 --- a/internal/api/client/filters/v1/filter_test.go +++ b/internal/api/client/filters/v1/filter_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" filtersV1 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v1" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/filters/v2/filter_test.go b/internal/api/client/filters/v2/filter_test.go index 8249546fb..20411c090 100644 --- a/internal/api/client/filters/v2/filter_test.go +++ b/internal/api/client/filters/v2/filter_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" filtersV2 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v2" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/followedtags/followedtags_test.go b/internal/api/client/followedtags/followedtags_test.go index 883ab033b..89a61aca1 100644 --- a/internal/api/client/followedtags/followedtags_test.go +++ b/internal/api/client/followedtags/followedtags_test.go @@ -21,6 +21,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/followedtags" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -79,6 +80,7 @@ func (suite *FollowedTagsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/followrequests/followrequest_test.go b/internal/api/client/followrequests/followrequest_test.go index fc9843b4a..1faac2bbc 100644 --- a/internal/api/client/followrequests/followrequest_test.go +++ b/internal/api/client/followrequests/followrequest_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -82,6 +83,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go index 8bfe444e5..293c96020 100644 --- a/internal/api/client/instance/instance_test.go +++ b/internal/api/client/instance/instance_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -84,6 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/lists/lists_test.go b/internal/api/client/lists/lists_test.go index 844d54cbb..ea5adbb5c 100644 --- a/internal/api/client/lists/lists_test.go +++ b/internal/api/client/lists/lists_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/lists" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -85,6 +86,7 @@ func (suite *ListsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/mutes/mutes_test.go b/internal/api/client/mutes/mutes_test.go index 5d450e32c..b721b080f 100644 --- a/internal/api/client/mutes/mutes_test.go +++ b/internal/api/client/mutes/mutes_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/mutes" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -81,6 +82,7 @@ func (suite *MutesTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/notifications/notifications_test.go b/internal/api/client/notifications/notifications_test.go index 23af65cb4..39fb66691 100644 --- a/internal/api/client/notifications/notifications_test.go +++ b/internal/api/client/notifications/notifications_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/notifications" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -81,6 +82,7 @@ func (suite *NotificationsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/polls/polls_test.go b/internal/api/client/polls/polls_test.go index 5a3c83580..b5552f39e 100644 --- a/internal/api/client/polls/polls_test.go +++ b/internal/api/client/polls/polls_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/polls" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -76,6 +77,7 @@ func (suite *PollsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/reports/reports_test.go b/internal/api/client/reports/reports_test.go index b36017d69..031fd953a 100644 --- a/internal/api/client/reports/reports_test.go +++ b/internal/api/client/reports/reports_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/reports" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -76,6 +77,7 @@ func (suite *ReportsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/search/search_test.go b/internal/api/client/search/search_test.go index 5ba198062..fecb30d38 100644 --- a/internal/api/client/search/search_test.go +++ b/internal/api/client/search/search_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/search" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -80,6 +81,7 @@ func (suite *SearchStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/statuses/status_test.go b/internal/api/client/statuses/status_test.go index 1a92276a1..5a4473344 100644 --- a/internal/api/client/statuses/status_test.go +++ b/internal/api/client/statuses/status_test.go @@ -25,6 +25,7 @@ "strings" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -192,6 +193,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/streaming/streaming_test.go b/internal/api/client/streaming/streaming_test.go index acdcafd8a..03cd6f434 100644 --- a/internal/api/client/streaming/streaming_test.go +++ b/internal/api/client/streaming/streaming_test.go @@ -31,6 +31,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/streaming" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -92,6 +93,7 @@ func (suite *StreamingTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/tags/tags_test.go b/internal/api/client/tags/tags_test.go index 79c708b10..71a84435a 100644 --- a/internal/api/client/tags/tags_test.go +++ b/internal/api/client/tags/tags_test.go @@ -26,6 +26,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/tags" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" @@ -87,6 +88,7 @@ func (suite *TagsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index 808daf1a3..320c743f8 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/client/user" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -72,6 +73,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/fileserver/fileserver_test.go b/internal/api/fileserver/fileserver_test.go index e5f684d0c..af6f125dc 100644 --- a/internal/api/fileserver/fileserver_test.go +++ b/internal/api/fileserver/fileserver_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/fileserver" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -98,6 +99,7 @@ func (suite *FileserverTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") diff --git a/internal/api/wellknown/webfinger/webfinger_test.go b/internal/api/wellknown/webfinger/webfinger_test.go index 67ac5a64e..76c1fb5bb 100644 --- a/internal/api/wellknown/webfinger/webfinger_test.go +++ b/internal/api/wellknown/webfinger/webfinger_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -79,6 +80,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/api/wellknown/webfinger/webfingerget_test.go b/internal/api/wellknown/webfinger/webfingerget_test.go index ce9bc0ccf..b3aec57fe 100644 --- a/internal/api/wellknown/webfinger/webfingerget_test.go +++ b/internal/api/wellknown/webfinger/webfingerget_test.go @@ -39,6 +39,7 @@ "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -90,6 +91,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.federator.TransportController(), suite.tc), suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), diff --git a/internal/cleaner/media_test.go b/internal/cleaner/media_test.go index 6e653c07c..afa015783 100644 --- a/internal/cleaner/media_test.go +++ b/internal/cleaner/media_test.go @@ -26,6 +26,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" @@ -67,6 +68,7 @@ func (suite *MediaTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/internal/config/config.go b/internal/config/config.go index 2bf2a77ad..72154b3f2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -78,15 +78,17 @@ type Configuration struct { WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."` WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"` - InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."` - InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"` - InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"` - InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"` - InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"` - InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"` - InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."` - InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"` - InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."` + InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."` + InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"` + InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"` + InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"` + InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"` + InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"` + InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."` + InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"` + InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."` + InstanceSubscriptionsProcessFrom string `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."` + InstanceSubscriptionsProcessEvery time.Duration `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."` AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."` AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 97d96d1ba..8c2ae90de 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -58,13 +58,15 @@ WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", - InstanceFederationMode: InstanceFederationModeDefault, - InstanceFederationSpamFilter: false, - InstanceExposePeers: false, - InstanceExposeSuspended: false, - InstanceExposeSuspendedWeb: false, - InstanceDeliverToSharedInboxes: true, - InstanceLanguages: make(language.Languages, 0), + InstanceFederationMode: InstanceFederationModeDefault, + InstanceFederationSpamFilter: false, + InstanceExposePeers: false, + InstanceExposeSuspended: false, + InstanceExposeSuspendedWeb: false, + InstanceDeliverToSharedInboxes: true, + InstanceLanguages: make(language.Languages, 0), + InstanceSubscriptionsProcessFrom: "23:00", // 11pm, + InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. AccountsRegistrationOpen: false, AccountsReasonRequired: true, diff --git a/internal/config/flags.go b/internal/config/flags.go index f96709e70..6f0957c36 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -90,6 +90,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) { cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage")) cmd.Flags().Bool(InstanceDeliverToSharedInboxesFlag(), cfg.InstanceDeliverToSharedInboxes, fieldtag("InstanceDeliverToSharedInboxes", "usage")) cmd.Flags().StringSlice(InstanceLanguagesFlag(), cfg.InstanceLanguages.TagStrs(), fieldtag("InstanceLanguages", "usage")) + cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage")) + cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage")) // Accounts cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 625c4ea78..e1c41638c 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1000,6 +1000,62 @@ func GetInstanceLanguages() language.Languages { return global.GetInstanceLangua // SetInstanceLanguages safely sets the value for global configuration 'InstanceLanguages' field func SetInstanceLanguages(v language.Languages) { global.SetInstanceLanguages(v) } +// GetInstanceSubscriptionsProcessFrom safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field +func (st *ConfigState) GetInstanceSubscriptionsProcessFrom() (v string) { + st.mutex.RLock() + v = st.config.InstanceSubscriptionsProcessFrom + st.mutex.RUnlock() + return +} + +// SetInstanceSubscriptionsProcessFrom safely sets the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field +func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceSubscriptionsProcessFrom = v + st.reloadToViper() +} + +// InstanceSubscriptionsProcessFromFlag returns the flag name for the 'InstanceSubscriptionsProcessFrom' field +func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" } + +// GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field +func GetInstanceSubscriptionsProcessFrom() string { + return global.GetInstanceSubscriptionsProcessFrom() +} + +// SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field +func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) } + +// GetInstanceSubscriptionsProcessEvery safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field +func (st *ConfigState) GetInstanceSubscriptionsProcessEvery() (v time.Duration) { + st.mutex.RLock() + v = st.config.InstanceSubscriptionsProcessEvery + st.mutex.RUnlock() + return +} + +// SetInstanceSubscriptionsProcessEvery safely sets the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field +func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceSubscriptionsProcessEvery = v + st.reloadToViper() +} + +// InstanceSubscriptionsProcessEveryFlag returns the flag name for the 'InstanceSubscriptionsProcessEvery' field +func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" } + +// GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field +func GetInstanceSubscriptionsProcessEvery() time.Duration { + return global.GetInstanceSubscriptionsProcessEvery() +} + +// SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field +func SetInstanceSubscriptionsProcessEvery(v time.Duration) { + global.SetInstanceSubscriptionsProcessEvery(v) +} + // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) { st.mutex.RLock() diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index f00e876ae..9878a1b50 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -77,6 +78,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.client = testrig.NewMockHTTPClient(nil, "../../../testrig/media") suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage visFilter := visibility.NewFilter(&suite.state) diff --git a/internal/federation/federatingdb/federatingdb_test.go b/internal/federation/federatingdb/federatingdb_test.go index 360094887..2f07914ae 100644 --- a/internal/federation/federatingdb/federatingdb_test.go +++ b/internal/federation/federatingdb/federatingdb_test.go @@ -22,6 +22,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" @@ -91,6 +92,7 @@ func (suite *FederatingDBTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, suite.testAccounts) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) } func (suite *FederatingDBTestSuite) TearDownTest() { diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 0980bf295..f46f837da 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -53,6 +54,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/internal/oauth/clientstore_test.go b/internal/oauth/clientstore_test.go index fc3dabc3a..0ced34de6 100644 --- a/internal/oauth/clientstore_test.go +++ b/internal/oauth/clientstore_test.go @@ -22,6 +22,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/state" @@ -54,6 +55,7 @@ func (suite *PgClientStoreTestSuite) SetupTest() { testrig.InitTestConfig() suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 8eec1f9dd..7bd9658dc 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -22,6 +22,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -93,6 +94,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/processing/admin/account_test.go b/internal/processing/admin/account_test.go index 59b8afc77..baa6eb646 100644 --- a/internal/processing/admin/account_test.go +++ b/internal/processing/admin/account_test.go @@ -53,7 +53,7 @@ func (suite *AccountTestSuite) TestAccountActionSuspend() { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return suite.state.AdminActions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } diff --git a/internal/processing/admin/accountaction.go b/internal/processing/admin/accountaction.go index 59d4b420e..959f2cfcd 100644 --- a/internal/processing/admin/accountaction.go +++ b/internal/processing/admin/accountaction.go @@ -68,7 +68,7 @@ func (p *Processor) accountActionSuspend( ) (string, gtserror.WithCode) { actionID := id.NewULID() - errWithCode := p.actions.Run( + errWithCode := p.state.AdminActions.Run( ctx, >smodel.AdminAction{ ID: actionID, diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 170298ca5..08e6bf0d5 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -21,10 +21,10 @@ "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing/common" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -33,21 +33,14 @@ type Processor struct { // common processor logic c *common.Processor - state *state.State - cleaner *cleaner.Cleaner - converter *typeutils.Converter - federator *federation.Federator - media *media.Manager - transport transport.Controller - email email.Sender - - // admin Actions currently - // undergoing processing - actions *Actions -} - -func (p *Processor) Actions() *Actions { - return p.actions + state *state.State + cleaner *cleaner.Cleaner + subscriptions *subscriptions.Subscriptions + converter *typeutils.Converter + federator *federation.Federator + media *media.Manager + transport transport.Controller + email email.Sender } // New returns a new admin processor. @@ -55,6 +48,7 @@ func New( common *common.Processor, state *state.State, cleaner *cleaner.Cleaner, + subscriptions *subscriptions.Subscriptions, federator *federation.Federator, converter *typeutils.Converter, mediaManager *media.Manager, @@ -62,17 +56,14 @@ func New( emailSender email.Sender, ) Processor { return Processor{ - c: common, - state: state, - cleaner: cleaner, - converter: converter, - federator: federator, - media: mediaManager, - transport: transportController, - email: emailSender, - actions: &Actions{ - r: make(map[string]*gtsmodel.AdminAction), - state: state, - }, + c: common, + state: state, + cleaner: cleaner, + subscriptions: subscriptions, + converter: converter, + federator: federator, + media: mediaManager, + transport: transportController, + email: emailSender, } } diff --git a/internal/processing/admin/admin_test.go b/internal/processing/admin/admin_test.go index 3251264b6..f0839f2f6 100644 --- a/internal/processing/admin/admin_test.go +++ b/internal/processing/admin/admin_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + adminactions "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -33,6 +34,7 @@ "github.com/superseriousbusiness/gotosocial/internal/processing/admin" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -89,6 +91,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = adminactions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( @@ -109,6 +112,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.transportController, suite.tc), suite.tc, suite.federator, suite.oauthServer, diff --git a/internal/processing/admin/domainallow.go b/internal/processing/admin/domainallow.go index bab54e308..13f0307f2 100644 --- a/internal/processing/admin/domainallow.go +++ b/internal/processing/admin/domainallow.go @@ -22,14 +22,11 @@ "errors" "fmt" - "codeberg.org/gruf/go-kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/text" ) @@ -69,84 +66,30 @@ func (p *Processor) createDomainAllow( } } - actionID := id.NewULID() + // Run admin action to process + // side effects of allow. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainAllow.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain allow side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.AdminActions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionSuspend, - AccountID: adminAcct.ID, - Text: domainAllow.PrivateComment, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain allow side effects") - defer func() { l.Info("finished processing domain allow side effects") }() - - return p.domainAllowSideEffects(ctx, domainAllow) - }, + action, + p.state.AdminActions.DomainAllowF(action.ID, domainAllow), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } apiDomainAllow, errWithCode := p.apiDomainPerm(ctx, domainAllow, false) if errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainAllow, actionID, nil -} - -func (p *Processor) domainAllowSideEffects( - ctx context.Context, - allow *gtsmodel.DomainAllow, -) gtserror.MultiError { - if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { - // We're running in allowlist mode, - // so there are no side effects to - // process here. - return nil - } - - // We're running in blocklist mode or - // some similar mode which necessitates - // domain allow side effects if a block - // was in place when the allow was created. - // - // So, check if there's a block. - block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs := gtserror.NewMultiError(1) - errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) - return errs - } - - if block == nil { - // No block? - // No problem! - return nil - } - - // There was a block, over which the new - // allow ought to take precedence. To account - // for this, just run side effects as though - // the domain was being unblocked, while - // leaving the existing block in place. - // - // Any accounts that were suspended by - // the block will be unsuspended and be - // able to interact with the instance again. - return p.domainUnblockSideEffects(ctx, block) + return apiDomainAllow, action.ID, nil } func (p *Processor) deleteDomainAllow( @@ -179,77 +122,23 @@ func (p *Processor) deleteDomainAllow( return nil, "", gtserror.NewErrorInternalError(err) } - actionID := id.NewULID() + // Run admin action to process + // side effects of unallow. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainAllow.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain unallow side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.AdminActions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domainAllow.Domain, - Type: gtsmodel.AdminActionUnsuspend, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domainAllow.Domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain unallow side effects") - defer func() { l.Info("finished processing domain unallow side effects") }() - - return p.domainUnallowSideEffects(ctx, domainAllow) - }, + action, + p.state.AdminActions.DomainUnallowF(action.ID, domainAllow), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainAllow, actionID, nil -} - -func (p *Processor) domainUnallowSideEffects( - ctx context.Context, - allow *gtsmodel.DomainAllow, -) gtserror.MultiError { - if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { - // We're running in allowlist mode, - // so there are no side effects to - // process here. - return nil - } - - // We're running in blocklist mode or - // some similar mode which necessitates - // domain allow side effects if a block - // was in place when the allow was removed. - // - // So, check if there's a block. - block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs := gtserror.NewMultiError(1) - errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) - return errs - } - - if block == nil { - // No block? - // No problem! - return nil - } - - // There was a block, over which the previous - // allow was taking precedence. Now that the - // allow has been removed, we should put the - // side effects of the block back in place. - // - // To do this, process the block side effects - // again as though the block were freshly - // created. This will mark all accounts from - // the blocked domain as suspended, and clean - // up their follows/following, media, etc. - return p.domainBlockSideEffects(ctx, block) + return apiDomainAllow, action.ID, nil } diff --git a/internal/processing/admin/domainblock.go b/internal/processing/admin/domainblock.go index 2fe10c97b..f8c1a6708 100644 --- a/internal/processing/admin/domainblock.go +++ b/internal/processing/admin/domainblock.go @@ -21,18 +21,12 @@ "context" "errors" "fmt" - "time" - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/ap" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" ) @@ -72,149 +66,31 @@ func (p *Processor) createDomainBlock( } } - actionID := id.NewULID() + // Run admin action to process + // side effects of block. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionSuspend, + AccountID: adminAcct.ID, + Text: domainBlock.PrivateComment, + } - // Process domain block side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.AdminActions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionSuspend, - AccountID: adminAcct.ID, - Text: domainBlock.PrivateComment, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - skip, err := p.skipBlockSideEffects(ctx, domain) - if err != nil { - return err - } - if skip != "" { - l.Infof("skipping domain block side effects: %s", skip) - return nil - } - - l.Info("processing domain block side effects") - defer func() { l.Info("finished processing domain block side effects") }() - - return p.domainBlockSideEffects(ctx, domainBlock) - }, + action, + p.state.AdminActions.DomainBlockF(action.ID, domainBlock), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } apiDomainBlock, errWithCode := p.apiDomainPerm(ctx, domainBlock, false) if errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainBlock, actionID, nil -} - -// skipBlockSideEffects checks if side effects of block creation -// should be skipped for the given domain, taking account of -// instance federation mode, and existence of any allows -// which ought to "shield" this domain from being blocked. -// -// If the caller should skip, the returned string will be non-zero -// and will be set to a reason why side effects should be skipped. -// -// - blocklist mode + allow exists: "..." (skip) -// - blocklist mode + no allow: "" (don't skip) -// - allowlist mode + allow exists: "" (don't skip) -// - allowlist mode + no allow: "" (don't skip) -func (p *Processor) skipBlockSideEffects( - ctx context.Context, - domain string, -) (string, gtserror.MultiError) { - var ( - skip string // Assume "" (don't skip). - errs gtserror.MultiError - ) - - // Never skip block side effects in allowlist mode. - fediMode := config.GetInstanceFederationMode() - if fediMode == config.InstanceFederationModeAllowlist { - return skip, errs - } - - // We know we're in blocklist mode. - // - // We want to skip domain block side - // effects if an allow is already - // in place which overrides the block. - - // Check if an explicit allow exists for this domain. - domainAllow, err := p.state.DB.GetDomainAllow(ctx, domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("error getting domain allow: %w", err) - return skip, errs - } - - if domainAllow != nil { - skip = "running in blocklist mode, and an explicit allow exists for this domain" - return skip, errs - } - - return skip, errs -} - -// domainBlockSideEffects processes the side effects of a domain block: -// -// 1. Strip most info away from the instance entry for the domain. -// 2. Pass each account from the domain to the processor for deletion. -// -// It should be called asynchronously, since it can take a while when -// there are many accounts present on the given domain. -func (p *Processor) domainBlockSideEffects( - ctx context.Context, - block *gtsmodel.DomainBlock, -) gtserror.MultiError { - var errs gtserror.MultiError - - // If we have an instance entry for this domain, - // update it with the new block ID and clear all fields - instance, err := p.state.DB.GetInstance(ctx, block.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("db error getting instance %s: %w", block.Domain, err) - return errs - } - - if instance != nil { - // We had an entry for this domain. - columns := stubbifyInstance(instance, block.ID) - if err := p.state.DB.UpdateInstance(ctx, instance, columns...); err != nil { - errs.Appendf("db error updating instance: %w", err) - return errs - } - } - - // For each account that belongs to this domain, - // process an account delete message to remove - // that account's posts, media, etc. - if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { - if err := p.state.Workers.Client.Process(ctx, &messages.FromClientAPI{ - APObjectType: ap.ActorPerson, - APActivityType: ap.ActivityDelete, - GTSModel: block, - Origin: account, - Target: account, - }); err != nil { - errs.Append(err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return apiDomainBlock, action.ID, nil } func (p *Processor) deleteDomainBlock( @@ -247,104 +123,23 @@ func (p *Processor) deleteDomainBlock( return nil, "", gtserror.NewErrorInternalError(err) } - actionID := id.NewULID() + // Run admin action to process + // side effects of unblock. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainBlock.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain unblock side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.AdminActions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domainBlock.Domain, - Type: gtsmodel.AdminActionUnsuspend, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domainBlock.Domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain unblock side effects") - defer func() { l.Info("finished processing domain unblock side effects") }() - - return p.domainUnblockSideEffects(ctx, domainBlock) - }, + action, + p.state.AdminActions.DomainUnblockF(action.ID, domainBlock), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainBlock, actionID, nil -} - -// domainUnblockSideEffects processes the side effects of undoing a -// domain block: -// -// 1. Mark instance entry as no longer suspended. -// 2. Mark each account from the domain as no longer suspended, if the -// suspension origin corresponds to the ID of the provided domain block. -// -// It should be called asynchronously, since it can take a while when -// there are many accounts present on the given domain. -func (p *Processor) domainUnblockSideEffects( - ctx context.Context, - block *gtsmodel.DomainBlock, -) gtserror.MultiError { - var errs gtserror.MultiError - - // Update instance entry for this domain, if we have it. - instance, err := p.state.DB.GetInstance(ctx, block.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("db error getting instance %s: %w", block.Domain, err) - } - - if instance != nil { - // We had an entry, update it to signal - // that it's no longer suspended. - instance.SuspendedAt = time.Time{} - instance.DomainBlockID = "" - if err := p.state.DB.UpdateInstance( - ctx, - instance, - "suspended_at", - "domain_block_id", - ); err != nil { - errs.Appendf("db error updating instance: %w", err) - return errs - } - } - - // Unsuspend all accounts whose suspension origin was this domain block. - if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { - if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() { - // Account wasn't suspended, nothing to do. - return - } - - if account.SuspensionOrigin != block.ID { - // Account was suspended, but not by - // this domain block, leave it alone. - return - } - - // Account was suspended by this domain - // block, mark it as unsuspended. - account.SuspendedAt = time.Time{} - account.SuspensionOrigin = "" - - if err := p.state.DB.UpdateAccount( - ctx, - account, - "suspended_at", - "suspension_origin", - ); err != nil { - errs.Appendf("db error updating account %s: %w", account.Username, err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return apiDomainBlock, action.ID, nil } diff --git a/internal/processing/admin/domainkeysexpire.go b/internal/processing/admin/domainkeysexpire.go index 9853becbd..0613f502d 100644 --- a/internal/processing/admin/domainkeysexpire.go +++ b/internal/processing/admin/domainkeysexpire.go @@ -19,7 +19,6 @@ import ( "context" - "time" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -39,47 +38,23 @@ func (p *Processor) DomainKeysExpire( adminAcct *gtsmodel.Account, domain string, ) (string, gtserror.WithCode) { - actionID := id.NewULID() + // Run admin action to process + // side effects of key expiry. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionExpireKeys, + AccountID: adminAcct.ID, + } - // Process key expiration asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.AdminActions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionExpireKeys, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - return p.domainKeysExpireSideEffects(ctx, domain) - }, + action, + p.state.AdminActions.DomainKeysExpireF(domain), ); errWithCode != nil { - return actionID, errWithCode + return action.ID, errWithCode } - return actionID, nil -} - -func (p *Processor) domainKeysExpireSideEffects(ctx context.Context, domain string) gtserror.MultiError { - var ( - expiresAt = time.Now() - errs gtserror.MultiError - ) - - // For each account on this domain, expire - // the public key and update the account. - if err := p.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) { - account.PublicKeyExpiresAt = expiresAt - if err := p.state.DB.UpdateAccount(ctx, - account, - "public_key_expires_at", - ); err != nil { - errs.Appendf("db error updating account: %w", err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return action.ID, nil } diff --git a/internal/processing/admin/domainpermission_test.go b/internal/processing/admin/domainpermission_test.go index 5a73693db..c8f3560c3 100644 --- a/internal/processing/admin/domainpermission_test.go +++ b/internal/processing/admin/domainpermission_test.go @@ -186,7 +186,7 @@ func (suite *DomainBlockTestSuite) awaitAction(actionID string) { ctx := context.Background() if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return suite.state.AdminActions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } diff --git a/internal/processing/admin/domainpermissionsubscription.go b/internal/processing/admin/domainpermissionsubscription.go index 3d2f63d56..6c051222c 100644 --- a/internal/processing/admin/domainpermissionsubscription.go +++ b/internal/processing/admin/domainpermissionsubscription.go @@ -22,6 +22,7 @@ "errors" "fmt" "net/url" + "slices" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" @@ -283,3 +284,89 @@ func (p *Processor) DomainPermissionSubscriptionRemove( return p.apiDomainPermSub(ctx, permSub) } + +func (p *Processor) DomainPermissionSubscriptionTest( + ctx context.Context, + acct *gtsmodel.Account, + id string, +) (any, gtserror.WithCode) { + permSub, err := p.state.DB.GetDomainPermissionSubscriptionByID(ctx, id) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("db error getting domain permission subscription %s: %w", id, err) + return nil, gtserror.NewErrorInternalError(err) + } + + if permSub == nil { + err := fmt.Errorf("domain permission subscription %s not found", id) + return nil, gtserror.NewErrorNotFound(err, err.Error()) + } + + // To process the test/dry-run correctly, we need to get + // all domain perm subs of this type with a *higher* priority, + // to know whether we ought to create permissions or not. + permSubs, err := p.state.DB.GetDomainPermissionSubscriptionsByPriority( + ctx, + permSub.PermissionType, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("db error: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Find the index of the targeted + // subscription in the slice. + index := slices.IndexFunc( + permSubs, + func(ps *gtsmodel.DomainPermissionSubscription) bool { + return ps.ID == permSub.ID + }, + ) + + // Get a transport for calling permSub.URI. + tsport, err := p.transport.NewTransportForUsername(ctx, acct.Username) + if err != nil { + err := gtserror.Newf("error getting transport: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Everything *before* the targeted + // subscription has a higher priority. + higherPrios := permSubs[:index] + + // Call the permSub.URI and parse a list of perms from it. + // Any error returned here is a "real" one, not an error + // from fetching / parsing the list. + createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription( + ctx, + permSub, + tsport, + higherPrios, + true, // Dry run. + ) + if err != nil { + err := gtserror.Newf("error doing dry-run: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // If permSub has an error set on it now, + // we should return it to the caller. + if permSub.Error != "" { + return map[string]string{ + "error": permSub.Error, + }, nil + } + + // No error, so return the list of + // perms that would have been created. + apiPerms := make([]*apimodel.DomainPermission, 0, len(createdPerms)) + for _, perm := range createdPerms { + apiPerm, errWithCode := p.apiDomainPerm(ctx, perm, false) + if errWithCode != nil { + return nil, errWithCode + } + + apiPerms = append(apiPerms, apiPerm) + } + + return apiPerms, nil +} diff --git a/internal/processing/admin/util.go b/internal/processing/admin/util.go index aef435856..f04b3654b 100644 --- a/internal/processing/admin/util.go +++ b/internal/processing/admin/util.go @@ -19,86 +19,12 @@ import ( "context" - "errors" - "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -// stubbifyInstance renders the given instance as a stub, -// removing most information from it and marking it as -// suspended. -// -// For caller's convenience, this function returns the db -// names of all columns that are updated by it. -func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string { - instance.Title = "" - instance.SuspendedAt = time.Now() - instance.DomainBlockID = domainBlockID - instance.ShortDescription = "" - instance.Description = "" - instance.Terms = "" - instance.ContactEmail = "" - instance.ContactAccountUsername = "" - instance.ContactAccountID = "" - instance.Version = "" - - return []string{ - "title", - "suspended_at", - "domain_block_id", - "short_description", - "description", - "terms", - "contact_email", - "contact_account_username", - "contact_account_id", - "version", - } -} - -// rangeDomainAccounts iterates through all accounts -// originating from the given domain, and calls the -// provided range function on each account. -// -// If an error is returned while selecting accounts, -// the loop will stop and return the error. -func (p *Processor) rangeDomainAccounts( - ctx context.Context, - domain string, - rangeF func(*gtsmodel.Account), -) error { - var ( - limit = 50 // Limit selection to avoid spiking mem/cpu. - maxID string // Start with empty string to select from top. - ) - - for { - // Get (next) page of accounts. - accounts, err := p.state.DB.GetInstanceAccounts(ctx, domain, maxID, limit) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - // Real db error. - return gtserror.Newf("db error getting instance accounts: %w", err) - } - - if len(accounts) == 0 { - // No accounts left, we're done. - return nil - } - - // Set next max ID for paging down. - maxID = accounts[len(accounts)-1].ID - - // Call provided range function. - for _, account := range accounts { - rangeF(account) - } - } -} - // apiDomainPerm is a cheeky shortcut for returning // the API version of the given domain permission, // or an appropriate error if something goes wrong. diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index cc7ec617e..831ba1a43 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -103,6 +104,7 @@ func (suite *ConversationsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) suite.filter = visibility.NewFilter(&suite.state) diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go index 80f1a7be7..2930733c4 100644 --- a/internal/processing/media/media_test.go +++ b/internal/processing/media/media_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -75,6 +76,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/processing/processor.go b/internal/processing/processor.go index ce0f1cfb8..8dabfba96 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -48,6 +48,7 @@ "github.com/superseriousbusiness/gotosocial/internal/processing/user" "github.com/superseriousbusiness/gotosocial/internal/processing/workers" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -180,6 +181,7 @@ func (p *Processor) Workers() *workers.Processor { // NewProcessor returns a new Processor. func NewProcessor( cleaner *cleaner.Cleaner, + subscriptions *subscriptions.Subscriptions, converter *typeutils.Converter, federator *federation.Federator, oauthServer oauth.Server, @@ -210,7 +212,7 @@ func NewProcessor( // Instantiate the rest of the sub // processors + pin them to this struct. processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc) - processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender) + processor.admin = admin.New(&common, state, cleaner, subscriptions, federator, converter, mediaManager, federator.TransportController(), emailSender) processor.conversations = conversations.New(state, converter, visFilter) processor.fedi = fedi.New(state, &common, converter, federator, visFilter) processor.filtersv1 = filtersv1.New(state, converter, &processor.stream) diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index d0898a98d..f152f3fad 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -21,6 +21,7 @@ "context" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -34,6 +35,7 @@ "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/stream" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -102,6 +104,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.testActivities = testrig.NewTestActivities(suite.testAccounts) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage @@ -125,6 +128,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.transportController, suite.typeconverter), suite.typeconverter, suite.federator, suite.oauthServer, diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go index b3c446d14..74aef7188 100644 --- a/internal/processing/status/status_test.go +++ b/internal/processing/status/status_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -84,6 +85,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.typeConverter = typeutils.NewConverter(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.tc = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")) suite.storage = testrig.NewInMemoryStorage() diff --git a/internal/processing/stream/stream_test.go b/internal/processing/stream/stream_test.go index 2569ac701..96ea65b0f 100644 --- a/internal/processing/stream/stream_test.go +++ b/internal/processing/stream/stream_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" @@ -50,6 +51,7 @@ func (suite *StreamTestSuite) SetupTest() { suite.testTokens = testrig.NewTestTokens() suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.streamProcessor = stream.New(&suite.state, suite.oauthServer) diff --git a/internal/processing/timeline/timeline_test.go b/internal/processing/timeline/timeline_test.go index 593bfb8f3..8ff6be5d1 100644 --- a/internal/processing/timeline/timeline_test.go +++ b/internal/processing/timeline/timeline_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -55,6 +56,7 @@ func (suite *TimelineStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.timeline = timeline.New( &suite.state, diff --git a/internal/processing/user/user_test.go b/internal/processing/user/user_test.go index e473c5bb0..2a9e0a89f 100644 --- a/internal/processing/user/user_test.go +++ b/internal/processing/user/user_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -49,6 +50,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) diff --git a/internal/state/state.go b/internal/state/state.go index 90683acd4..8aefa658a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -19,6 +19,7 @@ import ( "codeberg.org/gruf/go-mutexes" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/storage" @@ -61,9 +62,14 @@ type State struct { // Storage provides access to the storage driver. Storage *storage.Driver - // Workers provides access to this state's collection of worker pools. + // Workers provides access to this + // state's collection of worker pools. Workers workers.Workers + // Struct to manage running admin + // actions (and locks thereupon). + AdminActions *admin.Actions + // prevent pass-by-value. _ nocopy } diff --git a/internal/subscriptions/domainperms.go b/internal/subscriptions/domainperms.go new file mode 100644 index 000000000..b1e22a0be --- /dev/null +++ b/internal/subscriptions/domainperms.go @@ -0,0 +1,811 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions + +import ( + "bufio" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "time" + + "codeberg.org/gruf/go-kv" + + "github.com/miekg/dns" + "github.com/superseriousbusiness/gotosocial/internal/admin" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// ScheduleJobs schedules domain permission subscription +// fetching + updating using configured parameters. +// +// Returns an error if `MediaCleanupFrom` +// is not a valid format (hh:mm:ss). +func (s *Subscriptions) ScheduleJobs() error { + const hourMinute = "15:04" + + var ( + now = time.Now() + processEvery = config.GetInstanceSubscriptionsProcessEvery() + processFromStr = config.GetInstanceSubscriptionsProcessFrom() + ) + + // Parse processFromStr as hh:mm. + // Resulting time will be on 1 Jan year zero. + processFrom, err := time.Parse(hourMinute, processFromStr) + if err != nil { + return gtserror.Newf( + "error parsing '%s' in time format 'hh:mm': %w", + processFromStr, err, + ) + } + + // Time travel from + // year zero, groovy. + firstProcessAt := time.Date( + now.Year(), + now.Month(), + now.Day(), + processFrom.Hour(), + processFrom.Minute(), + 0, + 0, + now.Location(), + ) + + // Ensure first processing is in the future. + for firstProcessAt.Before(now) { + firstProcessAt = firstProcessAt.Add(processEvery) + } + + fn := func(ctx context.Context, start time.Time) { + log.Info(ctx, "starting instance subscriptions processing") + + // In blocklist (default) mode, process allows + // first to provide immunity to block side effects. + // + // In allowlist mode, process blocks first to + // ensure allowlist doesn't override blocks. + var order [2]gtsmodel.DomainPermissionType + if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist { + order = [2]gtsmodel.DomainPermissionType{ + gtsmodel.DomainPermissionAllow, + gtsmodel.DomainPermissionBlock, + } + } else { + order = [2]gtsmodel.DomainPermissionType{ + gtsmodel.DomainPermissionBlock, + gtsmodel.DomainPermissionAllow, + } + } + + // Fetch + process subscribed perms in order. + for _, permType := range order { + s.ProcessDomainPermissionSubscriptions(ctx, permType) + } + + log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start)) + } + + log.Infof(nil, + "scheduling instance subscriptions processing to run every %s, starting from %s; next processing will run at %s", + processEvery, processFromStr, firstProcessAt, + ) + + // Schedule processing to execute according to schedule. + if !s.state.Workers.Scheduler.AddRecurring( + "@subsprocessing", + firstProcessAt, + processEvery, + fn, + ) { + panic("failed to schedule @subsprocessing") + } + + return nil +} + +// ProcessDomainPermissionSubscriptions processes all domain permission +// subscriptions of the given permission type by, in turn, calling the +// URI of each subscription, parsing the result into a list of domain +// permissions, and creating (or skipping) each permission as appropriate. +func (s *Subscriptions) ProcessDomainPermissionSubscriptions( + ctx context.Context, + permType gtsmodel.DomainPermissionType, +) { + log.Info(ctx, "start") + defer log.Info(ctx, "finished") + + // Get permission subscriptions in priority order (highest -> lowest). + permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + log.Errorf(ctx, "db error getting domain perm subs by priority: %v", err) + return + } + + if len(permSubs) == 0 { + // No subscriptions of this + // type, so nothing to do. + return + } + + // Get a transport using the instance account, + // we can reuse this for each HTTP call. + tsport, err := s.transportController.NewTransportForUsername(ctx, "") + if err != nil { + log.Errorf(ctx, "error getting transport for instance account: %v", err) + return + } + + for i, permSub := range permSubs { + // Higher priority permission subs = everything + // above this permission sub in the slice. + higherPrios := permSubs[:i] + + _, err := s.ProcessDomainPermissionSubscription( + ctx, + permSub, + tsport, + higherPrios, + false, // Not dry. Wet, if you will. + ) + if err != nil { + // Real db error. + log.Errorf(ctx, + "error processing domain permission subscription %s: %v", + permSub.URI, err, + ) + return + } + + // Update this perm sub. + err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub) + if err != nil { + // Real db error. + log.Errorf(ctx, "db error updating domain perm sub: %v", err) + return + } + } +} + +// ProcessDomainPermissionSubscription processes one domain permission +// subscription by dereferencing the URI, parsing the response into a list +// of permissions, and for each discovered permission either creating an +// entry in the database, or ignoring it if it's excluded or already +// covered by a higher-priority subscription. +// +// On success, the slice of discovered DomainPermissions will be returned. +// In case of parsing error, or error on the remote side, permSub.Error +// will be updated with the calling/parsing error, and `nil, nil` will be +// returned. In case of an actual db error, `nil, err` will be returned and +// the caller should handle it. +// +// getHigherPrios should be a function for returning a slice of domain +// permission subscriptions with a higher priority than the given permSub. +// +// If dry == true, then the URI will still be called, and permissions +// will be parsed, but they will not actually be created. +// +// Note that while this function modifies fields on the given permSub, +// it's up to the caller to update it in the database (if desired). +func (s *Subscriptions) ProcessDomainPermissionSubscription( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + tsport transport.Transport, + higherPrios []*gtsmodel.DomainPermissionSubscription, + dry bool, +) ([]gtsmodel.DomainPermission, error) { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"permType", permSub.PermissionType.String()}, + {"permSubURI", permSub.URI}, + }...) + + // Set FetchedAt as we're + // going to attempt this now. + permSub.FetchedAt = time.Now() + + // Call the URI, and only skip + // cache if we're doing a dry run. + resp, err := tsport.DereferenceDomainPermissions( + ctx, permSub, dry, + ) + if err != nil { + // Couldn't get this one, + // set error + return. + errStr := err.Error() + l.Warnf("couldn't dereference permSubURI: %+v", err) + permSub.Error = errStr + return nil, nil + } + + // If the permissions at URI weren't modified + // since last time, just update some metadata + // to indicate a successful fetch, and return. + if resp.Unmodified { + l.Debug("received 304 Not Modified from remote") + permSub.SuccessfullyFetchedAt = permSub.FetchedAt + if permSub.ETag == "" && resp.ETag != "" { + // We didn't have an ETag before but + // we have one now: probably the remote + // added ETag support in the meantime. + permSub.ETag = resp.ETag + } + return nil, nil + } + + // At this point we know we got a 200 OK + // from the URI, so we've got a live body! + // Try to parse the body as a list of wantedPerms + // that the subscription wants to create. + var wantedPerms []gtsmodel.DomainPermission + + switch permSub.ContentType { + + // text/csv + case gtsmodel.DomainPermSubContentTypeCSV: + wantedPerms, err = permsFromCSV(l, permSub.PermissionType, resp.Body) + + // application/json + case gtsmodel.DomainPermSubContentTypeJSON: + wantedPerms, err = permsFromJSON(l, permSub.PermissionType, resp.Body) + + // text/plain + case gtsmodel.DomainPermSubContentTypePlain: + wantedPerms, err = permsFromPlain(l, permSub.PermissionType, resp.Body) + } + + if err != nil { + // We retrieved the permissions from remote but + // the connection died halfway through transfer, + // or we couldn't parse the results, or something. + // Just set error and return. + errStr := err.Error() + l.Warnf("couldn't parse results: %+v", err) + permSub.Error = errStr + return nil, nil + } + + if len(wantedPerms) == 0 { + // Fetch was OK, and parsing was, on the surface at + // least, OK, but we didn't get any perms. Consider + // this an error as users will probably want to know. + const errStr = "fetch successful but parsed zero usable results" + l.Warn(errStr) + permSub.Error = errStr + return nil, nil + } + + // This can now be considered a successful fetch. + permSub.SuccessfullyFetchedAt = permSub.FetchedAt + permSub.ETag = resp.ETag + permSub.Error = "" + + // Keep track of which domain perms are + // created (or would be, if dry == true). + createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms)) + + // Iterate through wantedPerms and + // create (or dry create) each one. + for _, wantedPerm := range wantedPerms { + l = l.WithField("domain", wantedPerm.GetDomain()) + created, err := s.processDomainPermission( + ctx, l, + wantedPerm, + permSub, + higherPrios, + dry, + ) + if err != nil { + // Proper db error. + return nil, err + } + + if !created { + continue + } + + createdPerms = append(createdPerms, wantedPerm) + } + + return createdPerms, nil +} + +// processDomainPermission processes one wanted domain +// permission discovered via a domain permission sub's URI. +// +// Error will only be returned in case of an actual database +// error, else the error will be logged and nil returned. +func (s *Subscriptions) processDomainPermission( + ctx context.Context, + l log.Entry, + wantedPerm gtsmodel.DomainPermission, + permSub *gtsmodel.DomainPermissionSubscription, + higherPrios []*gtsmodel.DomainPermissionSubscription, + dry bool, +) (bool, error) { + // Set to true if domain permission + // actually (would be) created. + var created bool + + // If domain is excluded from automatic + // permission creation, don't process it. + domain := wantedPerm.GetDomain() + excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain) + if err != nil { + // Proper db error. + return created, err + } + + if excluded { + l.Debug("domain is excluded, skipping") + return created, nil + } + + // Check if a permission already exists for + // this domain, and if it's covered already + // by a higher-priority subscription. + existingPerm, covered, err := s.existingCovered( + ctx, permSub.PermissionType, domain, higherPrios, + ) + if err != nil { + // Proper db error. + return created, err + } + + if covered { + l.Debug("domain is covered by a higher-priority subscription, skipping") + return created, nil + } + + // At this point we know we + // should create the perm. + created = true + + if dry { + // Don't do creation or side + // effects if we're dry running. + return created, nil + } + + // Handle perm creation differently depending + // on whether or not a perm already existed. + existing := !util.IsNil(existingPerm) + switch { + + case !existing && *permSub.AsDraft: + // No existing perm, create as draft. + err = s.state.DB.PutDomainPermissionDraft( + ctx, + >smodel.DomainPermissionDraft{ + ID: id.NewULID(), + PermissionType: permSub.PermissionType, + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + }, + ) + + case !existing && !*permSub.AsDraft: + // No existing perm, create a new one of the + // appropriate type, and process side effects. + var ( + insertF func() error + action *gtsmodel.AdminAction + actionF admin.ActionF + ) + + if permSub.PermissionType == gtsmodel.DomainPermissionBlock { + // Prepare to insert + process a block. + domainBlock := >smodel.DomainBlock{ + ID: id.NewULID(), + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + } + insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) } + + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionSuspend, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.AdminActions.DomainBlockF(action.ID, domainBlock) + + } else { + // Prepare to insert + process an allow. + domainAllow := >smodel.DomainAllow{ + ID: id.NewULID(), + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + } + insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) } + + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.AdminActions.DomainAllowF(action.ID, domainAllow) + } + + // Insert the new perm in the db. + if err = insertF(); err != nil { + // Couldn't insert wanted perm, + // don't process side effects. + break + } + + // Run admin action to process + // side effects of permission. + err = s.state.AdminActions.Run(ctx, action, actionF) + + case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans: + // Perm exists but we should adopt/take + // it by copying over desired fields. + existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID()) + existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount()) + existingPerm.SetSubscriptionID(permSub.ID) + existingPerm.SetObfuscate(wantedPerm.GetObfuscate()) + existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment()) + existingPerm.SetPublicComment(wantedPerm.GetPublicComment()) + + switch p := existingPerm.(type) { + case *gtsmodel.DomainBlock: + err = s.state.DB.UpdateDomainBlock(ctx, p) + case *gtsmodel.DomainAllow: + err = s.state.DB.UpdateDomainAllow(ctx, p) + } + + default: + // Perm exists but we should leave it alone. + l.Debug("domain is covered by a higher-priority subscription, skipping") + } + + if err != nil && !errors.Is(err, db.ErrAlreadyExists) { + // Proper db error. + return created, err + } + + created = true + return created, nil +} + +func permsFromCSV( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + csvReader := csv.NewReader(body) + + // Read and validate column headers. + columnHeaders, err := csvReader.Read() + if err != nil { + body.Close() + return nil, gtserror.NewfAt(3, "error decoding csv column headers: %w", err) + } + + if !slices.Equal( + columnHeaders, + []string{ + "#domain", + "#severity", + "#reject_media", + "#reject_reports", + "#public_comment", + "#obfuscate", + }, + ) { + body.Close() + err := gtserror.NewfAt(3, "unexpected column headers in csv: %+v", columnHeaders) + return nil, err + } + + // Read remaining CSV records. + records, err := csvReader.ReadAll() + + // Totally done + // with body now. + body.Close() + + // Check for decode error. + if err != nil { + err := gtserror.NewfAt(3, "error decoding body into csv: %w", err) + return nil, err + } + + // Make sure we actually + // have some records. + if len(records) == 0 { + return nil, nil + } + + // Convert records to permissions slice. + perms := make([]gtsmodel.DomainPermission, 0, len(records)) + for _, record := range records { + if len(record) != 6 { + l.Warnf("skipping invalid-length record: %+v", record) + continue + } + + var ( + domainRaw = record[0] + severity = record[1] + publicComment = record[4] + obfuscateStr = record[5] + ) + + if severity != "suspend" { + l.Warnf("skipping non-suspend record: %+v", record) + continue + } + + obfuscate, err := strconv.ParseBool(obfuscateStr) + if err != nil { + l.Warnf("couldn't parse obfuscate field of record: %+v", record) + continue + } + + // Normalize + validate domain. + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + switch permType { + case gtsmodel.DomainPermissionBlock: + perm = >smodel.DomainBlock{Domain: domain} + case gtsmodel.DomainPermissionAllow: + perm = >smodel.DomainAllow{Domain: domain} + } + + // Set remaining fields. + perm.SetPublicComment(publicComment) + perm.SetObfuscate(&obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func permsFromJSON( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + var ( + dec = json.NewDecoder(body) + apiPerms = make([]*apimodel.DomainPermission, 0) + ) + + // Read body into memory as + // slice of domain permissions. + if err := dec.Decode(&apiPerms); err != nil { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "error decoding into json: %w", err) + } + + // Perform a secondary decode just to ensure we drained the + // entirety of the data source. Error indicates either extra + // trailing garbage, or multiple JSON values (invalid data). + if err := dec.Decode(&struct{}{}); err != io.EOF { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "data remaining after json") + } + + // Done with body. + _ = body.Close() + + // Convert apimodel perms to barebones internal perms. + perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms)) + for _, apiPerm := range apiPerms { + + // Normalize + validate domain. + domainRaw := apiPerm.Domain.Domain + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + switch permType { + case gtsmodel.DomainPermissionBlock: + perm = >smodel.DomainBlock{Domain: domain} + case gtsmodel.DomainPermissionAllow: + perm = >smodel.DomainAllow{Domain: domain} + } + + // Set remaining fields. + perm.SetPublicComment(apiPerm.PublicComment) + perm.SetObfuscate(&apiPerm.Obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func permsFromPlain( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + // Scan + split by line. + sc := bufio.NewScanner(body) + + // Read into domains + // line by line. + var domains []string + for sc.Scan() { + domains = append(domains, sc.Text()) + } + + // Whatever happened, we're + // done with the body now. + body.Close() + + // Check if error reading body. + if err := sc.Err(); err != nil { + return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err) + } + + // Convert raw domains to permissions. + perms := make([]gtsmodel.DomainPermission, 0, len(domains)) + for _, domainRaw := range domains { + + // Normalize + validate domain. + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + switch permType { + case gtsmodel.DomainPermissionBlock: + perm = >smodel.DomainBlock{Domain: domain} + case gtsmodel.DomainPermissionAllow: + perm = >smodel.DomainAllow{Domain: domain} + } + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func validateDomain(domain string) (string, error) { + // Basic validation. + if _, ok := dns.IsDomainName(domain); !ok { + err := fmt.Errorf("invalid domain name") + return "", err + } + + // Convert to punycode. + domain, err := util.Punify(domain) + if err != nil { + err := fmt.Errorf("could not punify domain: %w", err) + return "", err + } + + // Check for invalid characters + // after the punification process. + if strings.ContainsAny(domain, "*, \n") { + err := fmt.Errorf("invalid char(s) in domain") + return "", err + } + + return domain, nil +} + +func (s *Subscriptions) existingCovered( + ctx context.Context, + permType gtsmodel.DomainPermissionType, + domain string, + higherPrios []*gtsmodel.DomainPermissionSubscription, +) ( + existingPerm gtsmodel.DomainPermission, + covered bool, + err error, +) { + // Check for existing perm + // of appropriate type. + var dbErr error + switch permType { + case gtsmodel.DomainPermissionBlock: + existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain) + case gtsmodel.DomainPermissionAllow: + existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain) + } + + if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) { + // Real db error. + err = dbErr + return + } + + if util.IsNil(existingPerm) { + // Can't be covered if + // no existing perm. + return + } + + subscriptionID := existingPerm.GetSubscriptionID() + if subscriptionID == "" { + // Can't be covered if + // no subscription ID. + return + } + + // Covered if subscription ID is in the slice + // of higher-priority permission subscriptions. + covered = slices.ContainsFunc( + higherPrios, + func(permSub *gtsmodel.DomainPermissionSubscription) bool { + return permSub.ID == subscriptionID + }, + ) + + return +} diff --git a/internal/subscriptions/subscriptions.go b/internal/subscriptions/subscriptions.go new file mode 100644 index 000000000..3826cf185 --- /dev/null +++ b/internal/subscriptions/subscriptions.go @@ -0,0 +1,42 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions + +import ( + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +type Subscriptions struct { + state *state.State + transportController transport.Controller + tc *typeutils.Converter +} + +func New( + state *state.State, + transportController transport.Controller, + tc *typeutils.Converter, +) *Subscriptions { + return &Subscriptions{ + state: state, + transportController: transportController, + tc: tc, + } +} diff --git a/internal/subscriptions/subscriptions_test.go b/internal/subscriptions/subscriptions_test.go new file mode 100644 index 000000000..0d3003a79 --- /dev/null +++ b/internal/subscriptions/subscriptions_test.go @@ -0,0 +1,538 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" + "github.com/superseriousbusiness/gotosocial/internal/util" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +const ( + rMediaPath = "../../testrig/media" + rTemplatePath = "../../web/template" +) + +type SubscriptionsTestSuite struct { + suite.Suite + + testAccounts map[string]*gtsmodel.Account +} + +func (suite *SubscriptionsTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() + suite.testAccounts = testrig.NewTestAccounts() +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have some perms now. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("bigbums6969", permSub.ETag) + suite.EqualValues(3, count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a JSON list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.json", + ContentType: gtsmodel.DomainPermSubContentTypeJSON, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have some perms now. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("don't modify me daddy", permSub.ETag) + suite.EqualValues(3, count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plain list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have some perms now. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("this is a legit etag i swear", permSub.ETag) + suite.EqualValues(3, count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + // Include the ETag so it gets sent with the request. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + ETag: "bigbums6969", + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now NOT have blocks for the domains + // on the list, as the remote will have returned + // 304, indicating we should do nothing. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + _, err := testStructs.State.DB.GetDomainBlock(ctx, domain) + if !errors.Is(err, db.ErrNoEntries) { + suite.FailNowf("", "domain perm %s created when it shouldn't be") + } + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have no perms. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("bigbums6969", permSub.ETag) + suite.Zero(count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocks404() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + // The endpoint will return a 404 so we can test erroring. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/does_not_exist.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have no perms. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Zero(count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plaintext list of baddies, + // but try to parse as CSV content type (shouldn't work). + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have no perms. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Zero(count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`ProcessDomainPermissionSubscription: unexpected column headers in csv: [bumfaces.net]`, permSub.Error) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plaintext list of baddies, + // but try to parse as CSV content type (shouldn't work). + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + // Should have no perms. + count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Zero(count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error) +} + +func TestSubscriptionTestSuite(t *testing.T) { + suite.Run(t, new(SubscriptionsTestSuite)) +} diff --git a/internal/transport/derefdomainpermlist.go b/internal/transport/derefdomainpermlist.go new file mode 100644 index 000000000..c81117bc6 --- /dev/null +++ b/internal/transport/derefdomainpermlist.go @@ -0,0 +1,121 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transport + +import ( + "context" + "io" + "net/http" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type DereferenceDomainPermissionsResp struct { + // Set only if response was 200 OK. + // It's up to the caller to close + // this when they're done with it. + Body io.ReadCloser + + // True if response + // was 304 Not Modified. + Unmodified bool + + // May be set + // if 200 or 304. + ETag string +} + +func (t *transport) DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + skipCache bool, +) (*DereferenceDomainPermissionsResp, error) { + // Prepare new HTTP request to endpoint + req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil) + if err != nil { + return nil, err + } + + // Set basic auth header if necessary. + if permSub.FetchUsername != "" || permSub.FetchPassword != "" { + req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword) + } + + // Set relevant Accept headers. + // Allow fallback in case target doesn't + // negotiate content type correctly. + req.Header.Add("Accept-Charset", "utf-8") + req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*") + + // If skipCache is true, we want to skip setting Cache + // headers so that we definitely don't get a 304 back. + if !skipCache { + // If we've successfully fetched this list + // before, set If-Modified-Since to last + // success to make the request conditional. + // + // See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since + if !permSub.SuccessfullyFetchedAt.IsZero() { + timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat) + req.Header.Add("If-Modified-Since", timeStr) + } + + // If we've got an ETag stored for this list, set + // If-None-Match to make the request conditional. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources. + if len(permSub.ETag) != 0 { + req.Header.Add("If-None-Match", permSub.ETag) + } + } + + // Perform the HTTP request + rsp, err := t.GET(req) + if err != nil { + return nil, err + } + + // If we have an unexpected / error response, + // wrap + return as error. This will also drain + // and close the response body for us. + if rsp.StatusCode != http.StatusOK && + rsp.StatusCode != http.StatusNotModified { + err := gtserror.NewFromResponse(rsp) + return nil, err + } + + // Check already if we were given an ETag + // we can use, as ETag is often returned + // even on 304 Not Modified responses. + permsResp := &DereferenceDomainPermissionsResp{ + ETag: rsp.Header.Get("Etag"), + } + + if rsp.StatusCode == http.StatusNotModified { + // Nothing has changed on the remote side + // since we last fetched, so there's nothing + // to do and we don't need to read the body. + rsp.Body.Close() + permsResp.Unmodified = true + } else { + // Return the live body to the caller. + permsResp.Body = rsp.Body + } + + return permsResp, nil +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 7f7e985fc..45d43ff18 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -78,6 +78,20 @@ type Transport interface { // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) + // DereferenceDomainPermissions dereferences the + // permissions list present at the given permSub's URI. + // + // If "force", then If-Modified-Since and If-None-Match + // headers will *NOT* be sent with the outgoing request. + // + // If err == nil and Unmodified == false, then it's up + // to the caller to close the returned io.ReadCloser. + DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, + ) (*DereferenceDomainPermissionsResp, error) + // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error) } diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 3a884d53f..c51c0755f 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -21,6 +21,7 @@ "context" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index 0676bea1b..a5fe5201b 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -499,6 +500,7 @@ func (suite *TypeUtilsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.AdminActions = admin.New(suite.state.DB, &suite.state.Workers) storage := testrig.NewInMemoryStorage() suite.state.Storage = storage diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index a90e88a70..2af479125 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -2119,7 +2119,9 @@ func (c *Converter) DomainPermToAPIDomainPerm( domainPerm.PrivateComment = d.GetPrivateComment() domainPerm.SubscriptionID = d.GetSubscriptionID() domainPerm.CreatedBy = d.GetCreatedByAccountID() - domainPerm.CreatedAt = util.FormatISO8601(d.GetCreatedAt()) + if createdAt := d.GetCreatedAt(); !createdAt.IsZero() { + domainPerm.CreatedAt = util.FormatISO8601(createdAt) + } // If this is a draft, also add the permission type. if _, ok := d.(*gtsmodel.DomainPermissionDraft); ok { diff --git a/mkdocs.yml b/mkdocs.yml index 1b12db12b..82bb03791 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,6 +138,7 @@ nav: - "admin/signups.md" - "admin/federation_modes.md" - "admin/domain_blocks.md" + - "admin/domain_permission_subscriptions.md" - "admin/request_filtering_modes.md" - "admin/robots.md" - "admin/cli.md" diff --git a/test/envparsing.sh b/test/envparsing.sh index 68ad24d14..94d0855ca 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -115,6 +115,8 @@ EXPECT=$(cat << "EOF" "nl", "en-GB" ], + "instance-subscriptions-process-every": 86400000000000, + "instance-subscriptions-process-from": "23:00", "landing-page-user": "admin", "letsencrypt-cert-dir": "/gotosocial/storage/certs", "letsencrypt-email-address": "", diff --git a/testrig/config.go b/testrig/config.go index 673ed46b6..0a957a831 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -99,6 +99,8 @@ func testDefaults() config.Configuration { TagStr: "en-gb", }, }, + InstanceSubscriptionsProcessFrom: "23:00", // 11pm, + InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. AccountsRegistrationOpen: true, AccountsReasonRequired: true, diff --git a/testrig/processor.go b/testrig/processor.go index e098de33a..bbb8d9d1d 100644 --- a/testrig/processor.go +++ b/testrig/processor.go @@ -26,15 +26,27 @@ "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) // NewTestProcessor returns a Processor suitable for testing purposes. // The passed in state will have its worker functions set appropriately, // but the state will not be initialized. -func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor { +func NewTestProcessor( + state *state.State, + federator *federation.Federator, + emailSender email.Sender, + mediaManager *media.Manager, +) *processing.Processor { + return processing.NewProcessor( cleaner.New(state), + subscriptions.New( + state, + federator.TransportController(), + typeutils.NewConverter(state), + ), typeutils.NewConverter(state), federator, NewTestOauthServer(state.DB), diff --git a/testrig/teststructs.go b/testrig/teststructs.go index b88e37d55..58986bffa 100644 --- a/testrig/teststructs.go +++ b/testrig/teststructs.go @@ -18,6 +18,7 @@ package testrig import ( + "github.com/superseriousbusiness/gotosocial/internal/admin" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -25,6 +26,8 @@ "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/common" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" + "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -38,12 +41,13 @@ // and worker queues, which was causing issues // when running all tests at once. type TestStructs struct { - State *state.State - Common *common.Processor - Processor *processing.Processor - HTTPClient *MockHTTPClient - TypeConverter *typeutils.Converter - EmailSender email.Sender + State *state.State + Common *common.Processor + Processor *processing.Processor + HTTPClient *MockHTTPClient + TypeConverter *typeutils.Converter + EmailSender email.Sender + TransportController transport.Controller } func SetupTestStructs( @@ -56,6 +60,7 @@ func SetupTestStructs( db := NewTestDB(&state) state.DB = db + state.AdminActions = admin.New(db, &state.Workers) storage := NewInMemoryStorage() state.Storage = storage @@ -89,6 +94,7 @@ func SetupTestStructs( processor := processing.NewProcessor( cleaner.New(&state), + subscriptions.New(&state, transportController, typeconverter), typeconverter, federator, oauthServer, @@ -105,12 +111,13 @@ func SetupTestStructs( StandardStorageSetup(storage, rMediaPath) return &TestStructs{ - State: &state, - Common: &common, - Processor: processor, - HTTPClient: httpClient, - TypeConverter: typeconverter, - EmailSender: emailSender, + State: &state, + Common: &common, + Processor: processor, + HTTPClient: httpClient, + TypeConverter: typeconverter, + EmailSender: emailSender, + TransportController: transportController, } } diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go index 385c620db..8faed93ad 100644 --- a/testrig/transportcontroller.go +++ b/testrig/transportcontroller.go @@ -41,6 +41,8 @@ const ( applicationJSON = "application/json" applicationActivityJSON = "application/activity+json" + textCSV = "text/csv" + textPlain = "text/plain" ) // NewTestTransportController returns a test transport controller with the given http client. @@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat responseBytes = []byte(`{"error":"404 not found"}`) responseContentType = applicationJSON responseContentLength = len(responseBytes) + extraHeaders = make(map[string]string, 0) reqURLString = req.URL.String() ) @@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat responseContentType = applicationJSON responseContentLength = len(responseBytes) } else if strings.Contains(reqURLString, ".well-known/webfinger") { - responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req) } else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") { - responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req) } else if strings.Contains(reqURLString, ".well-known/host-meta") { - responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req) + } else if strings.Contains(reqURLString, "lists.example.org") { + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req) } else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok { // the request is for a note that we have stored noteI, err := streams.Serialize(note) @@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat } log.Debugf(nil, "returning response %s", string(responseBytes)) + reader := bytes.NewReader(responseBytes) readCloser := io.NopCloser(reader) + + header := http.Header{ + "Content-Type": {responseContentType}, + } + for k, v := range extraHeaders { + header.Add(k, v) + } + return &http.Response{ Request: req, StatusCode: responseCode, Body: readCloser, ContentLength: int64(responseContentLength), - Header: http.Header{"Content-Type": {responseContentType}}, + Header: header, }, nil } @@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) ( return m.do(req) } -func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) { +func HostMetaResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { var hm *apimodel.HostMeta if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" { @@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte return } -func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) { +func WebfingerResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { var wfr *apimodel.WellKnownResponse switch req.URL.String() { @@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt responseContentLength = len(wfrJSON) return } + +func DomainPermissionSubscriptionResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { + + const ( + csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate +bumfaces.net,suspend,false,false,big jerks,false +peepee.poopoo,suspend,false,false,harassment,false +nothanks.com,suspend,false,false,,false` + csvRespETag = "bigbums6969" + + textResp = `bumfaces.net +peepee.poopoo +nothanks.com` + textRespETag = "this is a legit etag i swear" + + jsonResp = `[ + { + "domain": "bumfaces.net", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "big jerks" + }, + { + "domain": "peepee.poopoo", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "harassment" + }, + { + "domain": "nothanks.com", + "suspended_at": "2020-05-13T13:29:12.000Z" + } +]` + jsonRespETag = "don't modify me daddy" + ) + + switch req.URL.String() { + case "https://lists.example.org/baddies.csv": + extraHeaders = map[string]string{"ETag": csvRespETag} + if req.Header.Get("If-None-Match") == csvRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(csvResp) + responseContentType = textCSV + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + case "https://lists.example.org/baddies.txt": + extraHeaders = map[string]string{"ETag": textRespETag} + if req.Header.Get("If-None-Match") == textRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(textResp) + responseContentType = textPlain + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + case "https://lists.example.org/baddies.json": + extraHeaders = map[string]string{"ETag": jsonRespETag} + if req.Header.Get("If-None-Match") == jsonRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(jsonResp) + responseContentType = applicationJSON + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + default: + responseCode = http.StatusNotFound + responseBytes = []byte(`{"error":"not found"}`) + responseContentType = applicationJSON + responseContentLength = len(responseBytes) + } + + return +} diff --git a/web/source/settings/components/pageable-list.tsx b/web/source/settings/components/pageable-list.tsx index 3571fb1cd..92c5e220d 100644 --- a/web/source/settings/components/pageable-list.tsx +++ b/web/source/settings/components/pageable-list.tsx @@ -30,7 +30,7 @@ export interface PageableListProps { items?: T[]; itemToEntry: (_item: T) => ReactNode; isLoading: boolean; - isFetching: boolean; + isFetching?: boolean; isError: boolean; error: FetchBaseQueryError | SerializedError | undefined; emptyMessage: ReactNode; diff --git a/web/source/settings/lib/query/admin/domain-permissions/subscriptions.ts b/web/source/settings/lib/query/admin/domain-permissions/subscriptions.ts index f065aaf54..fef512a56 100644 --- a/web/source/settings/lib/query/admin/domain-permissions/subscriptions.ts +++ b/web/source/settings/lib/query/admin/domain-permissions/subscriptions.ts @@ -20,6 +20,7 @@ import { gtsApi } from "../../gts-api"; import type { + DomainPerm, DomainPermSub, DomainPermSubCreateUpdateParams, DomainPermSubSearchParams, @@ -120,6 +121,13 @@ const extended = gtsApi.injectEndpoints({ asForm: true, body: { remove_children: remove_children }, }), + }), + + testDomainPermissionSubscription: build.mutation<{ error: string } | DomainPerm[], string>({ + query: (id) => ({ + method: "POST", + url: `/api/v1/admin/domain_permission_subscriptions/${id}/test`, + }), }) }), }); @@ -154,6 +162,11 @@ const useUpdateDomainPermissionSubscriptionMutation = extended.useUpdateDomainPe */ const useRemoveDomainPermissionSubscriptionMutation = extended.useRemoveDomainPermissionSubscriptionMutation; +/** + * Test a domain permission subscription to see if data can be fetched + parsed. + */ +const useTestDomainPermissionSubscriptionMutation = extended.useTestDomainPermissionSubscriptionMutation; + export { useLazySearchDomainPermissionSubscriptionsQuery, useGetDomainPermissionSubscriptionQuery, @@ -161,4 +174,5 @@ export { useGetDomainPermissionSubscriptionsPreviewQuery, useUpdateDomainPermissionSubscriptionMutation, useRemoveDomainPermissionSubscriptionMutation, + useTestDomainPermissionSubscriptionMutation, }; diff --git a/web/source/settings/style.css b/web/source/settings/style.css index bbb8fd61c..75e7e7e3f 100644 --- a/web/source/settings/style.css +++ b/web/source/settings/style.css @@ -1426,6 +1426,18 @@ button.tab-button { } } +.domain-permission-subscription-details { + > .list > .entries > .perm-preview { + gap: 0.5rem; + } + + > .perm-issue > b > code { + background: $info-bg; + padding: 0; + } +} + + .domain-permission-subscription-title { font-size: 1.2rem; font-weight: bold; @@ -1451,7 +1463,8 @@ button.tab-button { } } -.domain-permission-subscription-remove { +.domain-permission-subscription-remove, +.domain-permission-subscription-test { gap: 1rem; } diff --git a/web/source/settings/views/moderation/domain-permissions/subscriptions/detail.tsx b/web/source/settings/views/moderation/domain-permissions/subscriptions/detail.tsx index 408d81b92..06f1582ae 100644 --- a/web/source/settings/views/moderation/domain-permissions/subscriptions/detail.tsx +++ b/web/source/settings/views/moderation/domain-permissions/subscriptions/detail.tsx @@ -17,19 +17,20 @@ along with this program. If not, see . */ -import React, { useState } from "react"; +import React, { ReactNode, useState } from "react"; import { useLocation, useParams } from "wouter"; import { useBaseUrl } from "../../../../lib/navigation/util"; import BackButton from "../../../../components/back-button"; -import { useGetDomainPermissionSubscriptionQuery, useRemoveDomainPermissionSubscriptionMutation, useUpdateDomainPermissionSubscriptionMutation } from "../../../../lib/query/admin/domain-permissions/subscriptions"; +import { useGetDomainPermissionSubscriptionQuery, useRemoveDomainPermissionSubscriptionMutation, useTestDomainPermissionSubscriptionMutation, useUpdateDomainPermissionSubscriptionMutation } from "../../../../lib/query/admin/domain-permissions/subscriptions"; import { useBoolInput, useNumberInput, useTextInput } from "../../../../lib/form"; import FormWithData from "../../../../lib/form/form-with-data"; -import { DomainPermSub } from "../../../../lib/types/domain-permission"; +import { DomainPerm, DomainPermSub } from "../../../../lib/types/domain-permission"; import MutationButton from "../../../../components/form/mutation-button"; import { Checkbox, NumberInput, Select, TextInput } from "../../../../components/form/inputs"; import useFormSubmit from "../../../../lib/form/submit"; import UsernameLozenge from "../../../../components/username-lozenge"; import { urlValidator } from "../../../../lib/util/formvalidators"; +import { PageableList } from "../../../../components/pageable-list"; export default function DomainPermissionSubscriptionDetail() { const params = useParams(); @@ -56,6 +57,7 @@ function DomainPermSubForm({ data: permSub }: { data: DomainPermSub }) {

Domain Permission Subscription Detail

+ ); @@ -382,3 +384,73 @@ function DeleteDomainPermSub({ permSub, backLocation }: { permSub: DomainPermSub ); } + +function TestDomainPermSub({ permSub }: { permSub: DomainPermSub }) { + const permType = permSub.permission_type; + if (!permType) { + throw "permission_type was undefined"; + } + + const [ testSub, testRes ] = useTestDomainPermissionSubscriptionMutation(); + const onSubmit = (e) => { + e.preventDefault(); + testSub(permSub.id); + }; + + // Function to map an item to a list entry. + function itemToEntry(perm: DomainPerm): ReactNode { + return ( + + { perm.domain } + { perm.public_comment && <>({ perm.public_comment }) } + + ); + } + + return ( + <> +
+

Test Subscription

+ Click the "test" button to instruct your instance to do a test + fetch and parse of the {permType} list at the subscription URI. +
+ If the fetch is successful, you will see a list of {permType}s + (or {permType} drafts) that *would* be created by this subscription, + along with the public comment for each {permType} (if applicable). +
+ The test does not actually create those {permType}s in your database. + + + { testRes.data && "error" in testRes.data + ?
+ + + The following issue was encountered when doing a fetch + parse: +
{ testRes.data.error } +
This may be due to a temporary outage at the remote URL, + or you may wish to check your subscription settings and test again. + +
+ : <> + { testRes.data && `${testRes.data?.length} ${permType}s would be created by this subscription:`} + No entries!} + /> + + } + + ); +}