Mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2024-10-31 22:40:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

[feature] status refetch support (#1690)

Browse source 
* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* first draft of status refetching logic

Signed-off-by: kim <grufwub@gmail.com>

* fix testrig to use new federation alloc func signature

Signed-off-by: kim <grufwub@gmail.com>

* fix log format directive

Signed-off-by: kim <grufwub@gmail.com>

* add status fetched_at migration

Signed-off-by: kim <grufwub@gmail.com>

* remove unused / unchecked for error types

Signed-off-by: kim <grufwub@gmail.com>

* add back the used type...

Signed-off-by: kim <grufwub@gmail.com>

* add separate internal getStatus() function for derefThread() that doesn't recurse

Signed-off-by: kim <grufwub@gmail.com>

* improved mention and media attachment error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix log and error format directives

Signed-off-by: kim <grufwub@gmail.com>

* update account deref to match status deref changes

Signed-off-by: kim <grufwub@gmail.com>

* very small code formatting change to make things clearer

Signed-off-by: kim <grufwub@gmail.com>

* add more code comments

Signed-off-by: kim <grufwub@gmail.com>

* improved code commenting

Signed-off-by: kim <grufwub@gmail.com>

* only check for required further derefs if needed

Signed-off-by: kim <grufwub@gmail.com>

* improved cache invalidation

Signed-off-by: kim <grufwub@gmail.com>

* tweak cache restarting to use a (very small) backoff

Signed-off-by: kim <grufwub@gmail.com>

* small readability changes and fixes

Signed-off-by: kim <grufwub@gmail.com>

* fix account sync issues

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts + update account enrichment to accept already-passed accountable

Signed-off-by: kim <grufwub@gmail.com>

* remove secondary function declaration

Signed-off-by: kim <grufwub@gmail.com>

* normalise dereferencer get status / account behaviour, fix remaining tests

Signed-off-by: kim <grufwub@gmail.com>

* fix remaining rebase conflicts, finish commenting code

Signed-off-by: kim <grufwub@gmail.com>

* appease the linter

Signed-off-by: kim <grufwub@gmail.com>

* add source file header

Signed-off-by: kim <grufwub@gmail.com>

* update to use TIMESTAMPTZ column type instead of just TIMESTAMP

Signed-off-by: kim <grufwub@gmail.com>

* don't pass in 'updated_at' to UpdateEmoji()

Signed-off-by: kim <grufwub@gmail.com>

* use new ap.Resolve{Account,Status}able() functions

Signed-off-by: kim <grufwub@gmail.com>

* remove the somewhat confusing rescoping of the same variable names

Signed-off-by: kim <grufwub@gmail.com>

* update migration file name, improved database delete error returns

Signed-off-by: kim <grufwub@gmail.com>

* formatting

Signed-off-by: kim <grufwub@gmail.com>

* improved multi-delete database functions to minimise DB calls

Signed-off-by: kim <grufwub@gmail.com>

* remove unused type

Signed-off-by: kim <grufwub@gmail.com>

* fix delete statements

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-05-12 10:15:54 +01:00 committed by GitHub
parent ba5a464ca5
commit 6c9d8e78eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 1552 additions and 1118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/gotosocial/action/server/server.go
View file

@ -116,7 +116,7 @@
typeConverter := typeutils.NewConverter(dbService)
federatingDB := federatingdb.New(&state, typeConverter)
transportController := transport.NewController(&state, federatingDB, &federation.Clock{}, client)
federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
federator := federation.NewFederator(&state, federatingDB, transportController, typeConverter, mediaManager)
// decide whether to create a noop email sender (won't send emails) or a real one
var emailSender email.Sender

7
internal/cache/cache.go vendored
View file

@ -19,6 +19,7 @@
import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
type Caches struct {
@ -41,6 +42,8 @@ type Caches struct {
// Init will (re)initialize both the GTS and AP cache collections.
// NOTE: the cache MUST NOT be in use anywhere, this is not thread-safe.
func (c *Caches) Init() {
log.Infof(nil, "init: %p", c)
c.GTS.Init()
c.AP.Init()
c.Visibility.Init()
@ -52,6 +55,8 @@ func (c *Caches) Init() {
// Start will start both the GTS and AP cache collections.
func (c *Caches) Start() {
log.Infof(nil, "start: %p", c)
c.GTS.Start()
c.AP.Start()
c.Visibility.Start()
@ -59,6 +64,8 @@ func (c *Caches) Start() {
// Stop will stop both the GTS and AP cache collections.
func (c *Caches) Stop() {
log.Infof(nil, "stop: %p", c)
c.GTS.Stop()
c.AP.Stop()
c.Visibility.Stop()

6
internal/cache/util.go vendored
View file

@ -74,8 +74,14 @@ func tryStop[ValueType any](cache *result.Cache[ValueType], sweep time.Duration)
func tryUntil(msg string, count int, do func() bool) {
for i := 0; i < count; i++ {
if do() {
// success.
return
}
// Sleep for a little before retry (a bcakoff).
time.Sleep(time.Millisecond * 1 << (i + 1))
}
// panic on total failure as this shouldn't happen.
log.Panicf(nil, "failed %s after %d tries", msg, count)
}

31
internal/db/bundb/account.go
View file

@ -302,7 +302,7 @@ func (a *accountDB) UpdateAccount(ctx context.Context, account *gtsmodel.Account
columns = append(columns, "updated_at")
}
err := a.state.Caches.GTS.Account().Store(account, func() error {
return a.state.Caches.GTS.Account().Store(account, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
@ -338,15 +338,23 @@ func (a *accountDB) UpdateAccount(ctx context.Context, account *gtsmodel.Account
return err
})
})
if err != nil {
return err
}
return nil
}
func (a *accountDB) DeleteAccount(ctx context.Context, id string) db.Error {
if err := a.conn.RunInTx(ctx, func(tx bun.Tx) error {
defer a.state.Caches.GTS.Account().Invalidate("ID", id)
// Load account into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := a.GetAccountByID(gtscontext.SetBarebones(ctx), id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// NOTE: even if db.ErrNoEntries is returned, we
// still run the below transaction to ensure related
// objects are appropriately deleted.
return err
}
return a.conn.RunInTx(ctx, func(tx bun.Tx) error {
// clear out any emoji links
if _, err := tx.
NewDelete().
@ -363,14 +371,7 @@ func (a *accountDB) DeleteAccount(ctx context.Context, id string) db.Error {
Where("? = ?", bun.Ident("account.id"), id).
Exec(ctx)
return err
}); err != nil {
return err
}
// Invalidate account from database lookups.
a.state.Caches.GTS.Account().Invalidate("ID", id)
return nil
})
}
func (a *accountDB) GetAccountLastPosted(ctx context.Context, accountID string, webOnly bool) (time.Time, db.Error) {

2
internal/db/bundb/bundb_test.go
View file

@ -66,9 +66,9 @@ func (suite *BunDBStandardTestSuite) SetupSuite() {
}
func (suite *BunDBStandardTestSuite) SetupTest() {
suite.state.Caches.Init()
testrig.InitTestConfig()
testrig.InitTestLog()
suite.state.Caches.Init()
suite.db = testrig.NewTestDB(&suite.state)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
}

53
internal/db/bundb/emoji.go
View file

@ -19,10 +19,12 @@
import (
"context"
"errors"
"strings"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
@ -56,24 +58,46 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error
}
func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) {
// Update the emoji's last-updated
emoji.UpdatedAt = time.Now()
if _, err := e.conn.
NewUpdate().
Model(emoji).
Where("? = ?", bun.Ident("emoji.id"), emoji.ID).
Column(columns...).
Exec(ctx); err != nil {
return nil, e.conn.ProcessError(err)
if len(columns) > 0 {
// If we're updating by column, ensure "updated_at" is included.
columns = append(columns, "updated_at")
}
err := e.state.Caches.GTS.Emoji().Store(emoji, func() error {
_, err := e.conn.
NewUpdate().
Model(emoji).
Where("? = ?", bun.Ident("emoji.id"), emoji.ID).
Column(columns...).
Exec(ctx)
return e.conn.ProcessError(err)
})
if err != nil {
return nil, err
}
e.state.Caches.GTS.Emoji().Invalidate("ID", emoji.ID)
return emoji, nil
}
func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
if err := e.conn.RunInTx(ctx, func(tx bun.Tx) error {
defer e.state.Caches.GTS.Emoji().Invalidate("ID", id)
// Load emoji into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := e.GetEmojiByID(
gtscontext.SetBarebones(ctx),
id,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// NOTE: even if db.ErrNoEntries is returned, we
// still run the below transaction to ensure related
// objects are appropriately deleted.
return err
}
return e.conn.RunInTx(ctx, func(tx bun.Tx) error {
// delete links between this emoji and any statuses that use it
if _, err := tx.
NewDelete().
@ -101,12 +125,7 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
}
return nil
}); err != nil {
return err
}
e.state.Caches.GTS.Emoji().Invalidate("ID", id)
return nil
})
}
func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) {

3
internal/db/bundb/errors.go
View file

@ -52,7 +52,8 @@ func processSQLiteError(err error) db.Error {
// Handle supplied error code:
switch sqliteErr.Code() {
case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
return db.ErrAlreadyExists
default:
return err

29
internal/db/bundb/media.go
View file

@ -19,9 +19,11 @@
import (
"context"
"errors"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
@ -103,17 +105,26 @@ func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAtt
}
func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
// Attempt to delete from database.
if _, err := m.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Where("? = ?", bun.Ident("media_attachment.id"), id).
Exec(ctx); err != nil {
return m.conn.ProcessError(err)
defer m.state.Caches.GTS.Media().Invalidate("ID", id)
// Load media into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := m.GetAttachmentByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate this media item from the cache.
m.state.Caches.GTS.Media().Invalidate("ID", id)
return nil
// Finally delete media from DB.
_, err = m.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Where("? = ?", bun.Ident("media_attachment.id"), id).
Exec(ctx)
return m.conn.ProcessError(err)
}
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {

29
internal/db/bundb/mention.go
View file

@ -19,6 +19,7 @@
import (
"context"
"errors"
"fmt"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -109,16 +110,24 @@ func (m *mentionDB) PutMention(ctx context.Context, mention *gtsmodel.Mention) e
}
func (m *mentionDB) DeleteMentionByID(ctx context.Context, id string) error {
if _, err := m.conn.
NewDelete().
Table("mentions").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return m.conn.ProcessError(err)
defer m.state.Caches.GTS.Mention().Invalidate("ID", id)
// Load mention into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := m.GetMention(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate mention from the lookup cache.
m.state.Caches.GTS.Mention().Invalidate("ID", id)
return nil
// Finally delete mention from DB.
_, err = m.conn.NewDelete().
Table("mentions").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return m.conn.ProcessError(err)
}

47
internal/db/bundb/migrations/20230511181430_add_status_fetched_at.go Normal file
View file

@ -0,0 +1,47 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package migrations
import (
"context"
"strings"
"github.com/uptrace/bun"
)
func init() {
up := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
_, err := tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? TIMESTAMPTZ", bun.Ident("statuses"), bun.Ident("fetched_at"))
if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) {
return err
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

107
internal/db/bundb/notification.go
View file

@ -22,6 +22,7 @@
"errors"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
@ -179,16 +180,26 @@ func (n *notificationDB) PutNotification(ctx context.Context, notif *gtsmodel.No
}
func (n *notificationDB) DeleteNotificationByID(ctx context.Context, id string) db.Error {
if _, err := n.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("notifications"), bun.Ident("notification")).
Where("? = ?", bun.Ident("notification.id"), id).
Exec(ctx); err != nil {
return n.conn.ProcessError(err)
defer n.state.Caches.GTS.Notification().Invalidate("ID", id)
// Load notif into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := n.GetNotificationByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
n.state.Caches.GTS.Notification().Invalidate("ID", id)
return nil
// Finally delete notif from DB.
_, err = n.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("notifications"), bun.Ident("notification")).
Where("? = ?", bun.Ident("notification.id"), id).
Exec(ctx)
return n.conn.ProcessError(err)
}
func (n *notificationDB) DeleteNotifications(ctx context.Context, types []string, targetAccountID string, originAccountID string) db.Error {
@ -196,56 +207,88 @@ func (n *notificationDB) DeleteNotifications(ctx context.Context, types []string
return errors.New("DeleteNotifications: one of targetAccountID or originAccountID must be set")
}
// Capture notification IDs in a RETURNING statement.
var ids []string
var notifIDs []string
q := n.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("notifications"), bun.Ident("notification")).
Returning("?", bun.Ident("id"))
NewSelect().
Column("id").
Table("notifications")
if len(types) > 0 {
q = q.Where("? IN (?)", bun.Ident("notification.notification_type"), bun.In(types))
q = q.Where("? IN (?)", bun.Ident("notification_type"), bun.In(types))
}
if targetAccountID != "" {
q = q.Where("? = ?", bun.Ident("notification.target_account_id"), targetAccountID)
q = q.Where("? = ?", bun.Ident("target_account_id"), targetAccountID)
}
if originAccountID != "" {
q = q.Where("? = ?", bun.Ident("notification.origin_account_id"), originAccountID)
q = q.Where("? = ?", bun.Ident("origin_account_id"), originAccountID)
}
if _, err := q.Exec(ctx, &ids); err != nil {
if _, err := q.Exec(ctx, &notifIDs); err != nil {
return n.conn.ProcessError(err)
}
// Invalidate each returned ID.
for _, id := range ids {
n.state.Caches.GTS.Notification().Invalidate("ID", id)
defer func() {
// Invalidate all IDs on return.
for _, id := range notifIDs {
n.state.Caches.GTS.Notification().Invalidate("ID", id)
}
}()
// Load all notif into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range notifIDs {
_, err := n.GetNotificationByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := n.conn.NewDelete().
Table("notifications").
Where("? IN (?)", bun.Ident("id"), bun.In(notifIDs)).
Exec(ctx)
return n.conn.ProcessError(err)
}
func (n *notificationDB) DeleteNotificationsForStatus(ctx context.Context, statusID string) db.Error {
// Capture notification IDs in a RETURNING statement.
var ids []string
var notifIDs []string
q := n.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("notifications"), bun.Ident("notification")).
Where("? = ?", bun.Ident("notification.status_id"), statusID).
Returning("?", bun.Ident("id"))
NewSelect().
Column("id").
Table("notifications").
Where("? = ?", bun.Ident("status_id"), statusID)
if _, err := q.Exec(ctx, &ids); err != nil {
if _, err := q.Exec(ctx, &notifIDs); err != nil {
return n.conn.ProcessError(err)
}
// Invalidate each returned ID.
for _, id := range ids {
n.state.Caches.GTS.Notification().Invalidate("ID", id)
defer func() {
// Invalidate all IDs on return.
for _, id := range notifIDs {
n.state.Caches.GTS.Notification().Invalidate("ID", id)
}
}()
// Load all notif into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range notifIDs {
_, err := n.GetNotificationByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := n.conn.NewDelete().
Table("notifications").
Where("? IN (?)", bun.Ident("id"), bun.In(notifIDs)).
Exec(ctx)
return n.conn.ProcessError(err)
}

90
internal/db/bundb/relationship_block.go
View file

@ -25,7 +25,6 @@
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/uptrace/bun"
)
@ -142,62 +141,65 @@ func (r *relationshipDB) getBlock(ctx context.Context, lookup string, dbQuery fu
}
func (r *relationshipDB) PutBlock(ctx context.Context, block *gtsmodel.Block) error {
err := r.state.Caches.GTS.Block().Store(block, func() error {
return r.state.Caches.GTS.Block().Store(block, func() error {
_, err := r.conn.NewInsert().Model(block).Exec(ctx)
return r.conn.ProcessError(err)
})
if err != nil {
return err
}
// Invalidate block origin account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", block.AccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", block.AccountID)
// Invalidate block target account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", block.TargetAccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", block.TargetAccountID)
return nil
}
func (r *relationshipDB) DeleteBlockByID(ctx context.Context, id string) error {
block, err := r.GetBlockByID(gtscontext.SetBarebones(ctx), id)
defer r.state.Caches.GTS.Block().Invalidate("ID", id)
// Load block into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetBlockByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
return r.deleteBlock(ctx, block)
// Finally delete block from DB.
_, err = r.conn.NewDelete().
Table("blocks").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteBlockByURI(ctx context.Context, uri string) error {
block, err := r.GetBlockByURI(gtscontext.SetBarebones(ctx), uri)
defer r.state.Caches.GTS.Block().Invalidate("URI", uri)
// Load block into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetBlockByURI(gtscontext.SetBarebones(ctx), uri)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
return r.deleteBlock(ctx, block)
}
func (r *relationshipDB) deleteBlock(ctx context.Context, block *gtsmodel.Block) error {
if _, err := r.conn.
NewDelete().
// Finally delete block from DB.
_, err = r.conn.NewDelete().
Table("blocks").
Where("? = ?", bun.Ident("id"), block.ID).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
}
// Invalidate block from cache lookups.
r.state.Caches.GTS.Block().Invalidate("ID", block.ID)
return nil
Where("? = ?", bun.Ident("uri"), uri).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteAccountBlocks(ctx context.Context, accountID string) error {
var blockIDs []string
// Get full list of IDs.
if err := r.conn.NewSelect().
Column("id").
Table("blocks").
ColumnExpr("?", bun.Ident("id")).
WhereOr("? = ? OR ? = ?",
bun.Ident("account_id"),
accountID,
@ -208,11 +210,27 @@ func (r *relationshipDB) DeleteAccountBlocks(ctx context.Context, accountID stri
return r.conn.ProcessError(err)
}
defer func() {
// Invalidate all IDs on return.
for _, id := range blockIDs {
r.state.Caches.GTS.Block().Invalidate("ID", id)
}
}()
// Load all blocks into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range blockIDs {
if err := r.DeleteBlockByID(ctx, id); err != nil {
log.Errorf(ctx, "error deleting block %q: %v", id, err)
_, err := r.GetBlockByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := r.conn.NewDelete().
Table("blocks").
Where("? IN (?)", bun.Ident("id"), bun.In(blockIDs)).
Exec(ctx)
return r.conn.ProcessError(err)
}

97
internal/db/bundb/relationship_follow.go
View file

@ -171,23 +171,10 @@ func (r *relationshipDB) getFollow(ctx context.Context, lookup string, dbQuery f
}
func (r *relationshipDB) PutFollow(ctx context.Context, follow *gtsmodel.Follow) error {
err := r.state.Caches.GTS.Follow().Store(follow, func() error {
return r.state.Caches.GTS.Follow().Store(follow, func() error {
_, err := r.conn.NewInsert().Model(follow).Exec(ctx)
return r.conn.ProcessError(err)
})
if err != nil {
return err
}
// Invalidate follow origin account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", follow.AccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", follow.AccountID)
// Invalidate follow target account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", follow.TargetAccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", follow.TargetAccountID)
return nil
}
func (r *relationshipDB) UpdateFollow(ctx context.Context, follow *gtsmodel.Follow, columns ...string) error {
@ -211,38 +198,58 @@ func (r *relationshipDB) UpdateFollow(ctx context.Context, follow *gtsmodel.Foll
}
func (r *relationshipDB) DeleteFollowByID(ctx context.Context, id string) error {
if _, err := r.conn.NewDelete().
Table("follows").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
defer r.state.Caches.GTS.Follow().Invalidate("ID", id)
// Load follow into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetFollowByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate follow from cache lookups.
r.state.Caches.GTS.Follow().Invalidate("ID", id)
return nil
// Finally delete follow from DB.
_, err = r.conn.NewDelete().
Table("follows").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteFollowByURI(ctx context.Context, uri string) error {
if _, err := r.conn.NewDelete().
Table("follows").
Where("? = ?", bun.Ident("uri"), uri).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
defer r.state.Caches.GTS.Follow().Invalidate("URI", uri)
// Load follow into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetFollowByURI(gtscontext.SetBarebones(ctx), uri)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate follow from cache lookups.
r.state.Caches.GTS.Follow().Invalidate("URI", uri)
return nil
// Finally delete follow from DB.
_, err = r.conn.NewDelete().
Table("follows").
Where("? = ?", bun.Ident("uri"), uri).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteAccountFollows(ctx context.Context, accountID string) error {
var followIDs []string
// Get full list of IDs.
if _, err := r.conn.
NewDelete().
NewSelect().
Column("id").
Table("follows").
WhereOr("? = ? OR ? = ?",
bun.Ident("account_id"),
@ -250,15 +257,31 @@ func (r *relationshipDB) DeleteAccountFollows(ctx context.Context, accountID str
bun.Ident("target_account_id"),
accountID,
).
Returning("?", bun.Ident("id")).
Exec(ctx, &followIDs); err != nil {
return r.conn.ProcessError(err)
}
// Invalidate each returned ID.
defer func() {
// Invalidate all IDs on return.
for _, id := range followIDs {
r.state.Caches.GTS.Follow().Invalidate("ID", id)
}
}()
// Load all follows into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range followIDs {
r.state.Caches.GTS.Follow().Invalidate("ID", id)
_, err := r.GetFollowByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := r.conn.NewDelete().
Table("follows").
Where("? IN (?)", bun.Ident("id"), bun.In(followIDs)).
Exec(ctx)
return r.conn.ProcessError(err)
}

136
internal/db/bundb/relationship_follow_req.go
View file

@ -149,23 +149,10 @@ func (r *relationshipDB) getFollowRequest(ctx context.Context, lookup string, db
}
func (r *relationshipDB) PutFollowRequest(ctx context.Context, follow *gtsmodel.FollowRequest) error {
err := r.state.Caches.GTS.FollowRequest().Store(follow, func() error {
return r.state.Caches.GTS.FollowRequest().Store(follow, func() error {
_, err := r.conn.NewInsert().Model(follow).Exec(ctx)
return r.conn.ProcessError(err)
})
if err != nil {
return err
}
// Invalidate follow request origin account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", follow.AccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", follow.AccountID)
// Invalidate follow request target account ID cached visibility.
r.state.Caches.Visibility.Invalidate("ItemID", follow.TargetAccountID)
r.state.Caches.Visibility.Invalidate("RequesterID", follow.TargetAccountID)
return nil
}
func (r *relationshipDB) UpdateFollowRequest(ctx context.Context, followRequest *gtsmodel.FollowRequest, columns ...string) error {
@ -221,6 +208,9 @@ func (r *relationshipDB) AcceptFollowRequest(ctx context.Context, sourceAccountI
return nil, err
}
// Invalidate follow request from cache lookups on return.
defer r.state.Caches.GTS.FollowRequest().Invalidate("ID", followReq.ID)
// Delete original follow request.
if _, err := r.conn.
NewDelete().
@ -230,9 +220,6 @@ func (r *relationshipDB) AcceptFollowRequest(ctx context.Context, sourceAccountI
return nil, r.conn.ProcessError(err)
}
// Invalidate follow request from cache lookups
r.state.Caches.GTS.FollowRequest().Invalidate("ID", followReq.ID)
// Delete original follow request notification
if err := r.state.DB.DeleteNotifications(ctx, []string{
string(gtsmodel.NotificationFollowRequest),
@ -244,15 +231,30 @@ func (r *relationshipDB) AcceptFollowRequest(ctx context.Context, sourceAccountI
}
func (r *relationshipDB) RejectFollowRequest(ctx context.Context, sourceAccountID string, targetAccountID string) db.Error {
// Get original follow request.
followReq, err := r.GetFollowRequest(ctx, sourceAccountID, targetAccountID)
defer r.state.Caches.GTS.FollowRequest().Invalidate("AccountID.TargetAccountID", sourceAccountID, targetAccountID)
// Load followreq into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetFollowRequest(gtscontext.SetBarebones(ctx),
sourceAccountID,
targetAccountID,
)
if err != nil {
return err
}
// Delete original follow request.
if err := r.DeleteFollowRequestByID(ctx, followReq.ID); err != nil {
return err
// Attempt to delete follow request.
if _, err = r.conn.NewDelete().
Table("follow_requests").
Where("? = ? AND ? = ?",
bun.Ident("account_id"),
sourceAccountID,
bun.Ident("target_account_id"),
targetAccountID,
).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
}
// Delete original follow request notification
@ -262,54 +264,90 @@ func (r *relationshipDB) RejectFollowRequest(ctx context.Context, sourceAccountI
}
func (r *relationshipDB) DeleteFollowRequestByID(ctx context.Context, id string) error {
if _, err := r.conn.NewDelete().
Table("follow_requests").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
defer r.state.Caches.GTS.FollowRequest().Invalidate("ID", id)
// Load followreq into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetFollowRequestByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate follow request from cache lookups.
r.state.Caches.GTS.FollowRequest().Invalidate("ID", id)
return nil
// Finally delete followreq from DB.
_, err = r.conn.NewDelete().
Table("follow_requests").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteFollowRequestByURI(ctx context.Context, uri string) error {
if _, err := r.conn.NewDelete().
Table("follow_requests").
Where("? = ?", bun.Ident("uri"), uri).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
defer r.state.Caches.GTS.FollowRequest().Invalidate("URI", uri)
// Load followreq into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetFollowRequestByURI(gtscontext.SetBarebones(ctx), uri)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate follow request from cache lookups.
r.state.Caches.GTS.FollowRequest().Invalidate("URI", uri)
return nil
// Finally delete followreq from DB.
_, err = r.conn.NewDelete().
Table("follow_requests").
Where("? = ?", bun.Ident("uri"), uri).
Exec(ctx)
return r.conn.ProcessError(err)
}
func (r *relationshipDB) DeleteAccountFollowRequests(ctx context.Context, accountID string) error {
var followIDs []string
var followReqIDs []string
// Get full list of IDs.
if _, err := r.conn.
NewDelete().
Table("follow_requests").
NewSelect().
Column("id").
Table("follow_requestss").
WhereOr("? = ? OR ? = ?",
bun.Ident("account_id"),
accountID,
bun.Ident("target_account_id"),
accountID,
).
Returning("?", bun.Ident("id")).
Exec(ctx, &followIDs); err != nil {
Exec(ctx, &followReqIDs); err != nil {
return r.conn.ProcessError(err)
}
// Invalidate each returned ID.
for _, id := range followIDs {
r.state.Caches.GTS.FollowRequest().Invalidate("ID", id)
defer func() {
// Invalidate all IDs on return.
for _, id := range followReqIDs {
r.state.Caches.GTS.FollowRequest().Invalidate("ID", id)
}
}()
// Load all followreqs into cache, this *really* isn't
// great but it is the only way we can ensure we invalidate
// all related caches correctly (e.g. visibility).
for _, id := range followReqIDs {
_, err := r.GetFollowRequestByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := r.conn.NewDelete().
Table("follow_requests").
Where("? IN (?)", bun.Ident("id"), bun.In(followReqIDs)).
Exec(ctx)
return r.conn.ProcessError(err)
}

28
internal/db/bundb/report.go
View file

@ -19,10 +19,12 @@
import (
"context"
"errors"
"fmt"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
@ -192,14 +194,24 @@ func (r *reportDB) UpdateReport(ctx context.Context, report *gtsmodel.Report, co
}
func (r *reportDB) DeleteReportByID(ctx context.Context, id string) db.Error {
if _, err := r.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("reports"), bun.Ident("report")).
Where("? = ?", bun.Ident("report.id"), id).
Exec(ctx); err != nil {
return r.conn.ProcessError(err)
defer r.state.Caches.GTS.Report().Invalidate("ID", id)
// Load status into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := r.GetReportByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
r.state.Caches.GTS.Report().Invalidate("ID", id)
return nil
// Finally delete report from DB.
_, err = r.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("reports"), bun.Ident("report")).
Where("? = ?", bun.Ident("report.id"), id).
Exec(ctx)
return r.conn.ProcessError(err)
}

186
internal/db/bundb/status.go
View file

@ -244,7 +244,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
}
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error {
err := s.state.Caches.GTS.Status().Store(status, func() error {
return s.state.Caches.GTS.Status().Store(status, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
@ -304,21 +304,6 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
return err
})
})
if err != nil {
return err
}
for _, id := range status.AttachmentIDs {
// Invalidate media attachments from cache.
//
// NOTE: this is needed due to the way in which
// we upload status attachments, and only after
// update them with a known status ID. This is
// not the case for header/avatar attachments.
s.state.Caches.GTS.Media().Invalidate("ID", id)
}
return nil
}
func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error {
@ -328,88 +313,91 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co
columns = append(columns, "updated_at")
}
if err := s.conn.RunInTx(ctx, func(tx bun.Tx) error {
// create links between this status and any emojis it uses
for _, i := range status.EmojiIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToEmoji{
StatusID: status.ID,
EmojiID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("emoji_id")).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// create links between this status and any tags it uses
for _, i := range status.TagIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToTag{
StatusID: status.ID,
TagID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("tag_id")).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// change the status ID of the media attachments to the new status
for _, a := range status.Attachments {
a.StatusID = status.ID
a.UpdatedAt = time.Now()
if _, err := tx.
NewUpdate().
Model(a).
Where("? = ?", bun.Ident("media_attachment.id"), a.ID).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// Finally, update the status
_, err := tx.
NewUpdate().
Model(status).
Column(columns...).
Where("? = ?", bun.Ident("status.id"), status.ID).
Exec(ctx)
return err
}); err != nil {
// already processed
return err
}
// Invalidate status from database lookups.
s.state.Caches.GTS.Status().Invalidate("ID", status.ID)
for _, id := range status.AttachmentIDs {
// Invalidate media attachments from cache.
return s.state.Caches.GTS.Status().Store(status, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
// NOTE: this is needed due to the way in which
// we upload status attachments, and only after
// update them with a known status ID. This is
// not the case for header/avatar attachments.
s.state.Caches.GTS.Media().Invalidate("ID", id)
}
return s.conn.RunInTx(ctx, func(tx bun.Tx) error {
// create links between this status and any emojis it uses
for _, i := range status.EmojiIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToEmoji{
StatusID: status.ID,
EmojiID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("emoji_id")).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
return nil
// create links between this status and any tags it uses
for _, i := range status.TagIDs {
if _, err := tx.
NewInsert().
Model(&gtsmodel.StatusToTag{
StatusID: status.ID,
TagID: i,
}).
On("CONFLICT (?, ?) DO NOTHING", bun.Ident("status_id"), bun.Ident("tag_id")).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// change the status ID of the media attachments to the new status
for _, a := range status.Attachments {
a.StatusID = status.ID
a.UpdatedAt = time.Now()
if _, err := tx.
NewUpdate().
Model(a).
Where("? = ?", bun.Ident("media_attachment.id"), a.ID).
Exec(ctx); err != nil {
err = s.conn.ProcessError(err)
if !errors.Is(err, db.ErrAlreadyExists) {
return err
}
}
}
// Finally, update the status
_, err := tx.
NewUpdate().
Model(status).
Column(columns...).
Where("? = ?", bun.Ident("status.id"), status.ID).
Exec(ctx)
return err
})
})
}
func (s *statusDB) DeleteStatusByID(ctx context.Context, id string) db.Error {
if err := s.conn.RunInTx(ctx, func(tx bun.Tx) error {
defer s.state.Caches.GTS.Status().Invalidate("ID", id)
// Load status into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := s.GetStatusByID(
gtscontext.SetBarebones(ctx),
id,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// NOTE: even if db.ErrNoEntries is returned, we
// still run the below transaction to ensure related
// objects are appropriately deleted.
return err
}
return s.conn.RunInTx(ctx, func(tx bun.Tx) error {
// delete links between this status and any emojis it uses
if _, err := tx.
NewDelete().
@ -438,17 +426,7 @@ func (s *statusDB) DeleteStatusByID(ctx context.Context, id string) db.Error {
}
return nil
}); err != nil {
return err
}
// Invalidate status from database lookups.
s.state.Caches.GTS.Status().Invalidate("ID", id)
// Invalidate status from all visibility lookups.
s.state.Caches.Visibility.Invalidate("ItemID", id)
return nil
})
}
func (s *statusDB) GetStatusParents(ctx context.Context, status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, db.Error) {

86
internal/db/bundb/statusfave.go
View file

@ -156,16 +156,26 @@ func (s *statusFaveDB) PutStatusFave(ctx context.Context, fave *gtsmodel.StatusF
}
func (s *statusFaveDB) DeleteStatusFaveByID(ctx context.Context, id string) db.Error {
if _, err := s.conn.
NewDelete().
Table("status_faves").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return s.conn.ProcessError(err)
defer s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
// Load fave into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := s.GetStatusFaveByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
return nil
// Finally delete fave from DB.
_, err = s.conn.NewDelete().
Table("status_faves").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return s.conn.ProcessError(err)
}
func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) db.Error {
@ -173,13 +183,12 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
return errors.New("DeleteStatusFaves: one of targetAccountID or originAccountID must be set")
}
// Capture fave IDs in a RETURNING statement.
var faveIDs []string
q := s.conn.
NewDelete().
Table("status_faves").
Returning("?", bun.Ident("id"))
NewSelect().
Column("id").
Table("status_faves")
if targetAccountID != "" {
q = q.Where("? = ?", bun.Ident("target_account_id"), targetAccountID)
@ -193,12 +202,29 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
return s.conn.ProcessError(err)
}
defer func() {
// Invalidate all IDs on return.
for _, id := range faveIDs {
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Load all faves into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
// Invalidate each of the returned status fave IDs.
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := s.conn.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.conn.ProcessError(err)
}
func (s *statusFaveDB) DeleteStatusFavesForStatus(ctx context.Context, statusID string) db.Error {
@ -206,19 +232,35 @@ func (s *statusFaveDB) DeleteStatusFavesForStatus(ctx context.Context, statusID
var faveIDs []string
q := s.conn.
NewDelete().
NewSelect().
Column("id").
Table("status_faves").
Where("? = ?", bun.Ident("status_id"), statusID).
Returning("?", bun.Ident("id"))
Where("? = ?", bun.Ident("status_id"), statusID)
if _, err := q.Exec(ctx, &faveIDs); err != nil {
return s.conn.ProcessError(err)
}
defer func() {
// Invalidate all IDs on return.
for _, id := range faveIDs {
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Load all faves into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
// Invalidate each of the returned status fave IDs.
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
return nil
// Finally delete all from DB.
_, err := s.conn.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.conn.ProcessError(err)
}

16
internal/db/bundb/tombstone.go
View file

@ -67,16 +67,12 @@ func (t *tombstoneDB) PutTombstone(ctx context.Context, tombstone *gtsmodel.Tomb
}
func (t *tombstoneDB) DeleteTombstone(ctx context.Context, id string) db.Error {
if _, err := t.conn.
NewDelete().
defer t.state.Caches.GTS.Tombstone().Invalidate("ID", id)
// Delete tombstone from DB.
_, err := t.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("tombstones"), bun.Ident("tombstone")).
Where("? = ?", bun.Ident("tombstone.id"), id).
Exec(ctx); err != nil {
return t.conn.ProcessError(err)
}
// Invalidate from cache by ID
t.state.Caches.GTS.Tombstone().Invalidate("ID", id)
return nil
Exec(ctx)
return t.conn.ProcessError(err)
}

50
internal/db/bundb/user.go
View file

@ -19,9 +19,11 @@
import (
"context"
"errors"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/uptrace/bun"
@ -155,32 +157,36 @@ func (u *userDB) UpdateUser(ctx context.Context, user *gtsmodel.User, columns ..
columns = append(columns, "updated_at")
}
// Update the user in DB
_, err := u.conn.
NewUpdate().
Model(user).
Where("? = ?", bun.Ident("user.id"), user.ID).
Column(columns...).
Exec(ctx)
if err != nil {
return u.state.Caches.GTS.User().Store(user, func() error {
_, err := u.conn.
NewUpdate().
Model(user).
Where("? = ?", bun.Ident("user.id"), user.ID).
Column(columns...).
Exec(ctx)
return u.conn.ProcessError(err)
}
// Invalidate user from cache
u.state.Caches.GTS.User().Invalidate("ID", user.ID)
return nil
})
}
func (u *userDB) DeleteUserByID(ctx context.Context, userID string) db.Error {
if _, err := u.conn.
NewDelete().
TableExpr("? AS ?", bun.Ident("users"), bun.Ident("user")).
Where("? = ?", bun.Ident("user.id"), userID).
Exec(ctx); err != nil {
return u.conn.ProcessError(err)
defer u.state.Caches.GTS.User().Invalidate("ID", userID)
// Load user into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := u.GetUserByID(gtscontext.SetBarebones(ctx), userID)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
err = nil
}
return err
}
// Invalidate user from cache
u.state.Caches.GTS.User().Invalidate("ID", userID)
return nil
// Finally delete user from DB.
_, err = u.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("users"), bun.Ident("user")).
Where("? = ?", bun.Ident("user.id"), userID).
Exec(ctx)
return u.conn.ProcessError(err)
}

438
internal/federation/dereferencing/account.go
View file

@ -38,7 +38,52 @@
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, error) {
// accountUpToDate returns whether the given account model is both updateable (i.e.
// non-instance remote account) and whether it needs an update based on `fetched_at`.
func accountUpToDate(account *gtsmodel.Account) bool {
if account.IsLocal() {
// Can't update local accounts.
return true
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return true
}
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(6 * time.Hour); time.Now().Before(next) {
return true
}
return false
}
// GetAccountByURI: implements Dereferencer{}.GetAccountByURI.
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error) {
// Fetch and dereference account if necessary.
account, apubAcc, err := d.getAccountByURI(ctx,
requestUser,
uri,
)
if err != nil {
return nil, nil, err
}
if apubAcc != nil {
// This account was updated, enqueue re-dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
}
return account, apubAcc, nil
}
// getAccountByURI is a package internal form of .GetAccountByURI() that doesn't bother dereferencing featured posts on update.
func (d *deref) getAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error) {
var (
account *gtsmodel.Account
uriStr = uri.String()
@ -46,23 +91,23 @@ func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *ur
)
// Search the database for existing account with ID URI.
account, err = d.db.GetAccountByURI(ctx, uriStr)
account, err = d.state.DB.GetAccountByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
return nil, nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
}
if account == nil {
// Else, search the database for existing by ID URL.
account, err = d.db.GetAccountByURL(ctx, uriStr)
account, err = d.state.DB.GetAccountByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
return nil, nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
}
}
if account == nil {
// Ensure that this is isn't a search for a local account.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for dereferencing.
@ -70,163 +115,193 @@ func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *ur
ID: id.NewULID(),
Domain: uri.Host,
URI: uriStr,
}, d.defaultFetchLatest, false)
}, nil)
}
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, uri, account, d.defaultFetchLatest, false)
// Check whether needs update.
if accountUpToDate(account) {
return account, nil, nil
}
// Try to update existing account model.
latest, apubAcc, err := d.enrichAccount(ctx,
requestUser,
uri,
account,
nil,
)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
return account, nil // fall back to returning existing
// Update fetch-at to slow re-attempts.
account.FetchedAt = time.Now()
_ = d.state.DB.UpdateAccount(ctx, account, "fetched_at")
// Fallback to existing.
return account, nil, nil
}
return enriched, nil
return latest, apubAcc, nil
}
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, error) {
// GetAccountByUsernameDomain: implements Dereferencer{}.GetAccountByUsernameDomain.
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, ap.Accountable, error) {
if domain == config.GetHost() || domain == config.GetAccountDomain() {
// We do local lookups using an empty domain,
// else it will fail the db search below.
domain = ""
}
// Search the database for existing account with USERNAME@DOMAIN
account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain)
// Search the database for existing account with USERNAME@DOMAIN.
account, err := d.state.DB.GetAccountByUsernameDomain(ctx, username, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
return nil, nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
}
if account == nil {
// Check for failed local lookup.
if domain == "" {
return nil, NewErrNotRetrievable(err) // wrapped err will be db.ErrNoEntries
// failed local lookup, will be db.ErrNoEntries.
return nil, nil, NewErrNotRetrievable(err)
}
// Create and pass-through a new bare-bones model for dereferencing.
account = &gtsmodel.Account{
account, apubAcc, err := d.enrichAccount(ctx, requestUser, nil, &gtsmodel.Account{
ID: id.NewULID(),
Username: username,
Domain: domain,
}, nil)
if err != nil {
return nil, nil, err
}
// There's no known account to fall back on,
// so return error if we can't enrich account.
return d.enrichAccount(ctx, requestUser, nil, account, d.defaultFetchLatest, false)
// This account was updated, enqueue dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
return account, apubAcc, nil
}
// We knew about this account already;
// try to update existing account model.
enriched, err := d.enrichAccount(ctx, requestUser, nil, account, d.defaultFetchLatest, false)
if err != nil {
log.Errorf(ctx, "error enriching account from remote: %v", err)
return account, nil // fall back to returning unchanged existing account model
}
return enriched, nil
}
func (d *deref) RefreshAccount(ctx context.Context, requestUser string, accountable ap.Accountable, account *gtsmodel.Account) (*gtsmodel.Account, error) {
// To avoid unnecessarily refetching multiple times from remote,
// we can just pass in the Accountable object that we received,
// if it was defined. If not, fall back to default fetch func.
var f fetchLatest
if accountable != nil {
f = func(
_ context.Context,
_ transport.Transport,
_ *url.URL,
_ string,
) (ap.Accountable, *gtsmodel.Account, error) {
return accountable, account, nil
}
} else {
f = d.defaultFetchLatest
}
// Set 'force' to 'true' to always fetch latest media etc.
return d.enrichAccount(ctx, requestUser, nil, account, f, true)
}
// fetchLatest defines a function for using a transport and uri to fetch the fetchLatest
// version of an account (and its AP representation) from a remote instance.
type fetchLatest func(ctx context.Context, transport transport.Transport, uri *url.URL, accountDomain string) (ap.Accountable, *gtsmodel.Account, error)
// defaultFetchLatest deduplicates latest fetching code that is used in several
// different functions. It simply calls the remote uri using the given transport,
// parses a returned AP representation into an account, and then returns both.
func (d *deref) defaultFetchLatest(ctx context.Context, transport transport.Transport, uri *url.URL, accountDomain string) (ap.Accountable, *gtsmodel.Account, error) {
// Dereference this account to get the latest available.
apubAcc, err := d.dereferenceAccountable(ctx, transport, uri)
if err != nil {
return nil, nil, fmt.Errorf("error dereferencing account %s: %w", uri, err)
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err := d.typeConverter.ASRepresentationToAccount(
ctx, apubAcc, accountDomain,
// Try to update existing account model.
latest, apubAcc, err := d.RefreshAccount(ctx,
requestUser,
account,
nil,
false,
)
if err != nil {
return nil, nil, fmt.Errorf("error converting accountable to gts model for account %s: %w", uri, err)
// Fallback to existing.
return account, nil, nil //nolint
}
return apubAcc, latestAcc, nil
return latest, apubAcc, nil
}
// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary.
func (d *deref) enrichAccount(
ctx context.Context,
requestUser string,
uri *url.URL,
account *gtsmodel.Account,
f fetchLatest,
force bool,
) (*gtsmodel.Account, error) {
if account.IsLocal() {
// Can't update local accounts.
return account, nil
// RefreshAccount: implements Dereferencer{}.RefreshAccount.
func (d *deref) RefreshAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) (*gtsmodel.Account, ap.Accountable, error) {
// Check whether needs update (and not forced).
if accountUpToDate(account) && !force {
return account, nil, nil
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return account, nil
}
if !force {
const interval = time.Hour * 48
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(interval); time.Now().Before(next) {
return account, nil
}
}
// Pre-fetch a transport for requesting username, used by later deref procedures.
transport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
// Parse the URI from account.
uri, err := url.Parse(account.URI)
if err != nil {
return nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
return nil, nil, fmt.Errorf("RefreshAccount: invalid account uri %q: %w", account.URI, err)
}
// Try to update + deref existing account model.
latest, apubAcc, err := d.enrichAccount(ctx,
requestUser,
uri,
account,
apubAcc,
)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
// Update fetch-at to slow re-attempts.
account.FetchedAt = time.Now()
_ = d.state.DB.UpdateAccount(ctx, account, "fetched_at")
return nil, nil, err
}
// This account was updated, enqueue re-dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
return latest, apubAcc, nil
}
// RefreshAccountAsync: implements Dereferencer{}.RefreshAccountAsync.
func (d *deref) RefreshAccountAsync(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) {
// Check whether needs update (and not forced).
if accountUpToDate(account) && !force {
return
}
// Parse the URI from account.
uri, err := url.Parse(account.URI)
if err != nil {
log.Errorf(ctx, "RefreshAccountAsync: invalid account uri %q: %v", account.URI, err)
return
}
// Enqueue a worker function to enrich this account async.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
latest, _, err := d.enrichAccount(ctx, requestUser, uri, account, apubAcc)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
return
}
// This account was updated, re-dereference account featured posts.
if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
}
// enrichAccount will enrich the given account, whether a new barebones model, or existing model from the database. It handles necessary dereferencing, webfingering etc.
func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, apubAcc ap.Accountable) (*gtsmodel.Account, ap.Accountable, error) {
// Pre-fetch a transport for requesting username, used by later deref procedures.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
accDomain, accURI, err := d.fingerRemoteAccount(ctx, transport, account.Username, account.Domain)
accDomain, accURI, err := d.fingerRemoteAccount(ctx, tsport, account.Username, account.Domain)
if err != nil {
if account.URI == "" {
// this is a new account (to us) with username@domain but failed webfinger, nothing more we can do.
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichAccount: error webfingering account: %w", err)}
}
switch {
case err != nil && account.URI == "":
// this is a new account (to us) with username@domain but failed webfinger, nothing more we can do.
return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err)
case err != nil:
// Simply log this error and move on, we already have an account URI.
log.Errorf(ctx, "error webfingering[1] remote account %s@%s: %v", account.Username, account.Domain, err)
}
case err == nil:
if err == nil {
if account.Domain != accDomain {
// Domain has changed, assume the activitypub
// account data provided may not be the latest.
apubAcc = nil
// After webfinger, we now have correct account domain from which we can do a final DB check.
alreadyAccount, err := d.db.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
alreadyAccount, err := d.state.DB.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("enrichAccount: db err looking for account again after webfinger: %w", err)
return nil, nil, fmt.Errorf("enrichAccount: db err looking for account again after webfinger: %w", err)
}
if err == nil {
if alreadyAccount != nil {
// Enrich existing account.
account = alreadyAccount
}
@ -240,30 +315,49 @@ func (d *deref) enrichAccount(
}
if uri == nil {
var err error
// No URI provided / found, must parse from account.
uri, err = url.Parse(account.URI)
if err != nil {
return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
return nil, nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
}
}
// Check whether this account URI is a blocked domain / subdomain.
if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err))
if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error checking blocked domain: %w", err)
} else if blocked {
return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
return nil, nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
}
// Mark deref+update handshake start.
d.startHandshake(requestUser, uri)
defer d.stopHandshake(requestUser, uri)
// Fetch latest version of the account, dereferencing if necessary.
apubAcc, latestAcc, err := f(ctx, transport, uri, account.Domain)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error calling fetchLatest function: %w", err)
// By default we assume that apubAcc has been passed,
// indicating that the given account is already latest.
latestAcc := account
if apubAcc == nil {
// Dereference latest version of the account.
b, err := tsport.Dereference(ctx, uri)
if err != nil {
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichAccount: error deferencing %s: %w", uri, err)}
}
// Attempt to resolve ActivityPub account from data.
apubAcc, err = ap.ResolveAccountable(ctx, b)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error resolving accountable from data for account %s: %w", uri, err)
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err = d.typeConverter.ASRepresentationToAccount(ctx,
apubAcc,
account.Domain,
)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err)
}
}
if account.Username == "" {
@ -281,11 +375,17 @@ func (d *deref) enrichAccount(
// Assume the host from the returned ActivityPub representation.
idProp := apubAcc.GetJSONLDId()
if idProp == nil || !idProp.IsIRI() {
return nil, errors.New("enrichAccount: no id property found on person, or id was not an iri")
return nil, nil, errors.New("enrichAccount: no id property found on person, or id was not an iri")
}
// Get IRI host value.
accHost := idProp.GetIRI().Host
accDomain, _, err := d.fingerRemoteAccount(ctx, transport, latestAcc.Username, accHost)
latestAcc.Domain, _, err = d.fingerRemoteAccount(ctx,
tsport,
latestAcc.Username,
accHost,
)
if err != nil {
// We still couldn't webfinger the account, so we're not certain
// what the accountDomain actually is. Still, we can make a solid
@ -293,9 +393,6 @@ func (d *deref) enrichAccount(
// If we're wrong, we can just try again in a couple days.
log.Errorf(ctx, "error webfingering[2] remote account %s@%s: %v", latestAcc.Username, accHost, err)
latestAcc.Domain = accHost
} else {
// Update account with latest info.
latestAcc.Domain = accDomain
}
}
@ -307,14 +404,15 @@ func (d *deref) enrichAccount(
latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID
latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID
if force || (latestAcc.AvatarRemoteURL != account.AvatarRemoteURL) {
if (latestAcc.AvatarMediaAttachmentID == "") ||
(latestAcc.AvatarRemoteURL != account.AvatarRemoteURL) {
// Reset the avatar media ID (handles removed).
latestAcc.AvatarMediaAttachmentID = ""
if latestAcc.AvatarRemoteURL != "" {
// Avatar has changed to a new one, fetch up-to-date copy and use new ID.
latestAcc.AvatarMediaAttachmentID, err = d.fetchRemoteAccountAvatar(ctx,
transport,
tsport,
latestAcc.AvatarRemoteURL,
latestAcc.ID,
)
@ -328,14 +426,15 @@ func (d *deref) enrichAccount(
}
}
if force || (latestAcc.HeaderRemoteURL != account.HeaderRemoteURL) {
if (latestAcc.HeaderMediaAttachmentID == "") ||
(latestAcc.HeaderRemoteURL != account.HeaderRemoteURL) {
// Reset the header media ID (handles removed).
latestAcc.HeaderMediaAttachmentID = ""
if latestAcc.HeaderRemoteURL != "" {
// Header has changed to a new one, fetch up-to-date copy and use new ID.
latestAcc.HeaderMediaAttachmentID, err = d.fetchRemoteAccountHeader(ctx,
transport,
tsport,
latestAcc.HeaderRemoteURL,
latestAcc.ID,
)
@ -363,15 +462,16 @@ func (d *deref) enrichAccount(
latestAcc.UpdatedAt = latestAcc.FetchedAt
// This is new, put it in the database.
err := d.db.PutAccount(ctx, latestAcc)
err := d.state.DB.PutAccount(ctx, latestAcc)
if errors.Is(err, db.ErrAlreadyExists) {
// TODO: replace this quick fix with per-URI deref locks.
latestAcc, err = d.db.GetAccountByURI(ctx, latestAcc.URI)
latestAcc, err = d.state.DB.GetAccountByURI(ctx, latestAcc.URI)
return latestAcc, nil, err
}
if err != nil {
return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
return nil, nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
}
} else {
// Set time of update from the last-fetched date.
@ -382,35 +482,12 @@ func (d *deref) enrichAccount(
latestAcc.Language = account.Language
// This is an existing account, update the model in the database.
if err := d.db.UpdateAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
if err := d.state.DB.UpdateAccount(ctx, latestAcc); err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
}
}
if latestAcc.FeaturedCollectionURI != "" {
// Fetch this account's pinned statuses, now that the account is in the database.
//
// The order is important here: if we tried to fetch the pinned statuses before
// storing the account, the process might end up calling enrichAccount again,
// causing us to get stuck in a loop. By calling it now, we make sure this doesn't
// happen!
if err := d.fetchRemoteAccountFeatured(ctx, requestUser, latestAcc.FeaturedCollectionURI, latestAcc.ID); err != nil {
log.Errorf(ctx, "error fetching featured collection for account %s: %v", uri, err)
}
}
return latestAcc, nil
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
// it finds as something that an account model can be constructed out of.
func (d *deref) dereferenceAccountable(ctx context.Context, transport transport.Transport, remoteAccountID *url.URL) (ap.Accountable, error) {
b, err := transport.Dereference(ctx, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("dereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err)
}
return ap.ResolveAccountable(ctx, b)
return latestAcc, apubAcc, nil
}
func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.Transport, avatarURL string, accountID string) (string, error) {
@ -531,7 +608,7 @@ func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gts
if len(maybeEmojiIDs) > len(maybeEmojis) {
maybeEmojis = make([]*gtsmodel.Emoji, 0, len(maybeEmojiIDs))
for _, emojiID := range maybeEmojiIDs {
maybeEmoji, err := d.db.GetEmojiByID(ctx, emojiID)
maybeEmoji, err := d.state.DB.GetEmojiByID(ctx, emojiID)
if err != nil {
return false, err
}
@ -631,18 +708,18 @@ func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gts
return changed, nil
}
// fetchRemoteAccountFeatured dereferences an account's featuredCollectionURI (if not empty).
// For each discovered status, this status will be dereferenced (if necessary) and marked as
// pinned (if necessary). Then, old pins will be removed if they're not included in new pins.
func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUsername string, featuredCollectionURI string, accountID string) error {
uri, err := url.Parse(featuredCollectionURI)
// dereferenceAccountFeatured dereferences an account's featuredCollectionURI (if not empty). For each discovered status, this status will
// be dereferenced (if necessary) and marked as pinned (if necessary). Then, old pins will be removed if they're not included in new pins.
func (d *deref) dereferenceAccountFeatured(ctx context.Context, requestUser string, account *gtsmodel.Account) error {
uri, err := url.Parse(account.FeaturedCollectionURI)
if err != nil {
return err
}
tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
// Pre-fetch a transport for requesting username, used by later deref procedures.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return err
return fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
b, err := tsport.Dereference(ctx, uri)
@ -661,7 +738,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
}
if t.GetTypeName() != ap.ObjectOrderedCollection {
return fmt.Errorf("%s was not an OrderedCollection", featuredCollectionURI)
return fmt.Errorf("%s was not an OrderedCollection", uri)
}
collection, ok := t.(vocab.ActivityStreamsOrderedCollection)
@ -675,7 +752,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
}
// Get previous pinned statuses (we'll need these later).
wasPinned, err := d.db.GetAccountPinnedStatuses(ctx, accountID)
wasPinned, err := d.state.DB.GetAccountPinnedStatuses(ctx, account.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return fmt.Errorf("error getting account pinned statuses: %w", err)
}
@ -720,11 +797,10 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
// we still know it was *meant* to be pinned.
statusURIs = append(statusURIs, statusURI)
status, _, err := d.GetStatus(ctx, requestingUsername, statusURI, false, false)
status, _, err := d.getStatusByURI(ctx, requestUser, statusURI)
if err != nil {
// We couldn't get the status, bummer.
// Just log + move on, we can try later.
log.Errorf(ctx, "error getting status from featured collection %s: %s", featuredCollectionURI, err)
// We couldn't get the status, bummer. Just log + move on, we can try later.
log.Errorf(ctx, "error getting status from featured collection %s: %v", statusURI, err)
continue
}
@ -733,7 +809,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
continue
}
if status.AccountID != accountID {
if status.AccountID != account.ID {
// Someone's pinned a status that doesn't
// belong to them, this doesn't work for us.
continue
@ -748,8 +824,9 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
// All conditions are met for this status to
// be pinned, so we can finally update it.
status.PinnedAt = time.Now()
if err := d.db.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error updating status in featured collection %s: %s", featuredCollectionURI, err)
if err := d.state.DB.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error updating status in featured collection %s: %v", status.URI, err)
continue
}
}
@ -768,8 +845,9 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
// Status was pinned before, but is not included
// in most recent pinned uris, so unpin it now.
status.PinnedAt = time.Time{}
if err := d.db.UpdateStatus(ctx, status, "pinned_at"); err != nil {
return fmt.Errorf("error unpinning status: %w", err)
if err := d.state.DB.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error unpinning status %s: %v", status.URI, err)
continue
}
}

20
internal/federation/dereferencing/account_test.go
View file

@ -37,7 +37,7 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
fetchingAccount := suite.testAccounts["local_account_1"]
groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
group, err := suite.dereferencer.GetAccountByURI(
group, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
groupURL,
@ -61,7 +61,7 @@ func (suite *AccountTestSuite) TestDereferenceService() {
fetchingAccount := suite.testAccounts["local_account_1"]
serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
service, err := suite.dereferencer.GetAccountByURI(
service, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
serviceURL,
@ -93,7 +93,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -112,7 +112,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb
suite.FailNow(err.Error())
}
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -126,7 +126,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -140,7 +140,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -154,7 +154,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
targetAccount.Username,
@ -168,7 +168,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
@ -183,7 +183,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername()
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
@ -198,7 +198,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDom
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),

9
internal/federation/dereferencing/announce.go
View file

@ -40,7 +40,7 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
}
// Check whether the originating status is from a blocked host
if blocked, err := d.db.IsDomainBlocked(ctx, boostedURI.Host); blocked || err != nil {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, boostedURI.Host); blocked || err != nil {
return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedURI.Host)
}
@ -48,7 +48,7 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
if boostedURI.Host == config.GetHost() {
// This is a local status, fetch from the database
status, err := d.db.GetStatusByURI(ctx, boostedURI.String())
status, err := d.state.DB.GetStatusByURI(ctx, boostedURI.String())
if err != nil {
return fmt.Errorf("DereferenceAnnounce: error fetching local status %q: %v", announce.BoostOf.URI, err)
}
@ -57,14 +57,11 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
boostedStatus = status
} else {
// This is a boost of a remote status, we need to dereference it.
status, statusable, err := d.GetStatus(ctx, requestingUsername, boostedURI, true, true)
status, _, err := d.GetStatusByURI(ctx, requestingUsername, boostedURI)
if err != nil {
return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.BoostOf.URI, err)
}
// Dereference all statuses in the thread of the boosted status
d.DereferenceThread(ctx, requestingUsername, boostedURI, status, statusable)
// Set boosted status
boostedStatus = status
}

6
internal/federation/dereferencing/collectionpage.go
View file

@ -29,9 +29,9 @@
"github.com/superseriousbusiness/gotosocial/internal/ap"
)
// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
func (d *deref) DereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
// dereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
func (d *deref) dereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host)
}

51
internal/federation/dereferencing/dereferencer.go
View file

@ -24,42 +24,61 @@
"codeberg.org/gruf/go-mutexes"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
type Dereferencer interface {
// GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the
// last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, error)
// GetAccountByURI will attempt to fetch an accounts by its URI, first checking the database. In the case of a newly-met remote model, or a remote model
// whose last_fetched date is beyond a certain interval, the account will be dereferenced. In the case of dereferencing, some low-priority account information
// may be enqueued for asynchronous fetching, e.g. featured account statuses (pins). An ActivityPub object indicates the account was dereferenced.
GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error)
// GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either
// check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, error)
// GetAccountByUsernameDomain will attempt to fetch an accounts by its username@domain, first checking the database. In the case of a newly-met remote model,
// or a remote model whose last_fetched date is beyond a certain interval, the account will be dereferenced. In the case of dereferencing, some low-priority
// account information may be enqueued for asynchronous fetching, e.g. featured account statuses (pins). An ActivityPub object indicates the account was dereferenced.
GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, ap.Accountable, error)
// RefreshAccount forces a refresh of the given account by fetching the current/latest state of the account from the remote instance.
// An updated account model is returned, but not yet inserted/updated in the database; this is the caller's responsibility.
RefreshAccount(ctx context.Context, requestUser string, accountable ap.Accountable, account *gtsmodel.Account) (*gtsmodel.Account, error)
// RefreshAccount updates the given account if remote and last_fetched is beyond fetch interval, or if force is set. An updated account model is returned,
// but in the case of dereferencing, some low-priority account information may be enqueued for asynchronous fetching, e.g. featured account statuses (pins).
// An ActivityPub object indicates the account was dereferenced (i.e. updated).
RefreshAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) (*gtsmodel.Account, ap.Accountable, error)
GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
// RefreshAccountAsync enqueues the given account for an asychronous update fetching, if last_fetched is beyond fetch interval, or if forcc is set.
// This is a more optimized form of manually enqueueing .UpdateAccount() to the federation worker, since it only enqueues update if necessary.
RefreshAccountAsync(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool)
// GetStatusByURI will attempt to fetch a status by its URI, first checking the database. In the case of a newly-met remote model, or a remote model
// whose last_fetched date is beyond a certain interval, the status will be dereferenced. In the case of dereferencing, some low-priority status information
// may be enqueued for asynchronous fetching, e.g. dereferencing the remainder of the status thread. An ActivityPub object indicates the status was dereferenced.
GetStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error)
// RefreshStatus updates the given status if remote and last_fetched is beyond fetch interval, or if force is set. An updated status model is returned,
// but in the case of dereferencing, some low-priority status information may be enqueued for asynchronous fetching, e.g. dereferencing the remainder of the
// status thread. An ActivityPub object indicates the status was dereferenced (i.e. updated).
RefreshStatus(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) (*gtsmodel.Status, ap.Statusable, error)
// RefreshStatusAsync enqueues the given status for an asychronous update fetching, if last_fetched is beyond fetch interval, or if force is set.
// This is a more optimized form of manually enqueueing .UpdateStatus() to the federation worker, since it only enqueues update if necessary.
RefreshStatusAsync(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable)
GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error)
Handshaking(username string, remoteAccountID *url.URL) bool
}
type deref struct {
db db.DB
state *state.State
typeConverter typeutils.TypeConverter
transportController transport.Controller
mediaManager media.Manager
@ -74,9 +93,9 @@ type deref struct {
}
// NewDereferencer returns a Dereferencer initialized with the given parameters.
func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
func NewDereferencer(state *state.State, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
return &deref{
db: db,
state: state,
typeConverter: typeConverter,
transportController: transportController,
mediaManager: mediaManager,

2
internal/federation/dereferencing/dereferencer_test.go
View file

@ -65,7 +65,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.state.DB = suite.db
suite.state.Storage = suite.storage
media := testrig.NewTestMediaManager(&suite.state)
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")), media)
suite.dereferencer = dereferencing.NewDereferencer(&suite.state, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")), media)
testrig.StandardDBSetup(suite.db, nil)
}

2
internal/federation/dereferencing/emoji.go
View file

@ -109,7 +109,7 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,
// it should be fleshed out already and we won't
// have to get it from the database again
gotEmoji = e
} else if gotEmoji, err = d.db.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {
} else if gotEmoji, err = d.state.DB.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {
log.Errorf(ctx, "error checking database for emoji %s: %s", shortcodeDomain, err)
continue
}

66
internal/federation/dereferencing/error.go
View file

@ -19,25 +19,8 @@
import (
"fmt"
"net/http"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
// ErrDB denotes that a proper error has occurred when doing
// a database call, as opposed to a simple db.ErrNoEntries.
type ErrDB struct {
wrapped error
}
func (err *ErrDB) Error() string {
return fmt.Sprintf("database error during dereferencing: %v", err.wrapped)
}
func newErrDB(err error) error {
return &ErrDB{wrapped: err}
}
// ErrNotRetrievable denotes that an item could not be dereferenced
// with the given parameters.
type ErrNotRetrievable struct {
@ -51,52 +34,3 @@ func (err *ErrNotRetrievable) Error() string {
func NewErrNotRetrievable(err error) error {
return &ErrNotRetrievable{wrapped: err}
}
// ErrTransportError indicates that something unforeseen went wrong creating
// a transport, or while making an http call to a remote resource with a transport.
type ErrTransportError struct {
wrapped error
}
func (err *ErrTransportError) Error() string {
return fmt.Sprintf("transport error: %v", err.wrapped)
}
func newErrTransportError(err error) error {
return &ErrTransportError{wrapped: err}
}
// ErrOther denotes some other kind of weird error, perhaps from a malformed json
// or some other weird crapola.
type ErrOther struct {
wrapped error
}
func (err *ErrOther) Error() string {
return fmt.Sprintf("unexpected error: %v", err.wrapped)
}
func newErrOther(err error) error {
return &ErrOther{wrapped: err}
}
func wrapDerefError(derefErr error, fluff string) error {
// Wrap with fluff.
err := derefErr
if fluff != "" {
err = fmt.Errorf("%s: %w", fluff, derefErr)
}
// Check for unretrievable HTTP status code errors.
if code := gtserror.StatusCode(derefErr); // nocollapse
code == http.StatusGone || code == http.StatusNotFound {
return NewErrNotRetrievable(err)
}
// Check for other untrievable errors.
if gtserror.NotFound(derefErr) {
return NewErrNotRetrievable(err)
}
return err
}

3
internal/federation/dereferencing/finger.go
View file

@ -69,6 +69,5 @@ func (d *deref) fingerRemoteAccount(ctx context.Context, transport transport.Tra
}
}
err = errors.New("fingerRemoteAccount: no match found in webfinger response")
return
return "", nil, errors.New("fingerRemoteAccount: no match found in webfinger response")
}

2
internal/federation/dereferencing/instance.go
View file

@ -26,7 +26,7 @@
)
func (d *deref) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, remoteInstanceURI.Host); blocked || err != nil {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, remoteInstanceURI.Host); blocked || err != nil {
return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
}

649
internal/federation/dereferencing/status.go
View file

@ -21,8 +21,9 @@
"context"
"errors"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
@ -34,374 +35,430 @@
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
// EnrichRemoteStatus takes a remote status that's already been inserted into the database in a minimal form,
// and populates it with additional fields, media, etc.
//
// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by
// the federatingDB's Create function, but additional dereferencing is needed on it.
func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) {
if err := d.populateStatusFields(ctx, status, username, includeParent); err != nil {
return nil, err
// statusUpToDate returns whether the given status model is both updateable
// (i.e. remote status) and whether it needs an update based on `fetched_at`.
func statusUpToDate(status *gtsmodel.Status) bool {
if *status.Local {
// Can't update local statuses.
return true
}
if err := d.db.UpdateStatus(ctx, status); err != nil {
return nil, err
// If this status was updated recently (last interval), we return as-is.
if next := status.FetchedAt.Add(2 * time.Hour); time.Now().Before(next) {
return true
}
return status, nil
return false
}
// GetStatus completely dereferences a status, converts it to a GtS model status,
// puts it in the database, and returns it to a caller.
//
// If refetch is true, then regardless of whether we have the original status in the database or not,
// the ap.Statusable representation of the status will be dereferenced and returned.
//
// If refetch is false, the ap.Statusable will only be returned if this is a new status, so callers
// should check whether or not this is nil.
//
// GetAccount will guard against trying to do http calls to fetch a status that belongs to this instance.
// Instead of making calls, it will just return the status early if it finds it, or return an error.
func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {
uriString := statusURI.String()
// GetStatus: implements Dereferencer{}.GetStatus().
func (d *deref) GetStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error) {
// Fetch and dereference status if necessary.
status, apubStatus, err := d.getStatusByURI(ctx,
requestUser,
uri,
)
if err != nil {
return nil, nil, err
}
// try to get by URI first
status, dbErr := d.db.GetStatusByURI(ctx, uriString)
if dbErr != nil {
if !errors.Is(dbErr, db.ErrNoEntries) {
// real error
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error during GetStatusByURI for %s: %w", uriString, dbErr))
if apubStatus != nil {
// This status was updated, enqueue re-dereferencing the whole thread.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
d.dereferenceThread(ctx, requestUser, uri, status, apubStatus)
})
}
return status, apubStatus, nil
}
// getStatusByURI is a package internal form of .GetStatusByURI() that doesn't bother dereferencing the whole thread on update.
func (d *deref) getStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error) {
var (
status *gtsmodel.Status
uriStr = uri.String()
err error
)
// Search the database for existing status with ID URI.
status, err = d.state.DB.GetStatusByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, nil, fmt.Errorf("GetStatusByURI: error checking database for status %s by uri: %w", uriStr, err)
}
if status == nil {
// Else, search the database for existing by ID URL.
status, err = d.state.DB.GetStatusByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, nil, fmt.Errorf("GetStatusByURI: error checking database for status %s by url: %w", uriStr, err)
}
// no problem, just press on
} else if !refetch {
// we already had the status and we aren't being asked to refetch the AP representation
}
if status == nil {
// Ensure that this is isn't a search for a local status.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for deref.
return d.enrichStatus(ctx, requestUser, uri, &gtsmodel.Status{
Local: func() *bool { var false bool; return &false }(),
URI: uriStr,
}, nil)
}
// Try to update + deref existing status model.
latest, apubStatus, err := d.enrichStatus(ctx,
requestUser,
uri,
status,
nil,
)
if err != nil {
log.Errorf(ctx, "error enriching remote status: %v", err)
// Update fetch-at to slow re-attempts.
status.FetchedAt = time.Now()
_ = d.state.DB.UpdateStatus(ctx, status, "fetched_at")
// Fallback to existing.
return status, nil, nil
}
// try to get by URL if we couldn't get by URI now
if status == nil {
status, dbErr = d.db.GetStatusByURL(ctx, uriString)
if dbErr != nil {
if !errors.Is(dbErr, db.ErrNoEntries) {
// real error
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error during GetStatusByURI for %s: %w", uriString, dbErr))
}
// no problem, just press on
} else if !refetch {
// we already had the status and we aren't being asked to refetch the AP representation
return status, nil, nil
}
}
// guard against having our own statuses passed in
if host := statusURI.Host; host == config.GetHost() || host == config.GetAccountDomain() {
// this is our status, definitely don't search for it
if status != nil {
return status, nil, nil
}
return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
}
// if we got here, either we didn't have the status
// in the db, or we had it but need to refetch it
tsport, err := d.transportController.NewTransportForUsername(ctx, username)
if err != nil {
return nil, nil, newErrTransportError(fmt.Errorf("GetRemoteStatus: error creating transport for %s: %w", username, err))
}
statusable, derefErr := d.dereferenceStatusable(ctx, tsport, statusURI)
if derefErr != nil {
return nil, nil, wrapDerefError(derefErr, "GetRemoteStatus: error dereferencing statusable")
}
if status != nil && refetch {
// we already had the status in the db, and we've also
// now fetched the AP representation as requested
return status, statusable, nil
}
// from here on out we can consider this to be a 'new' status because we didn't have the status in the db already
accountURI, err := ap.ExtractAttributedTo(statusable)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %w", err))
}
// we need to get the author of the status else we can't serialize it properly
if _, err = d.GetAccountByURI(ctx, username, accountURI); err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err))
}
status, err = d.typeConverter.ASStatusToStatus(ctx, statusable)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err))
}
ulid, err := id.NewULIDFromTime(status.CreatedAt)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err))
}
status.ID = ulid
if err := d.populateStatusFields(ctx, status, username, includeParent); err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err))
}
if err := d.db.PutStatus(ctx, status); err != nil && !errors.Is(err, db.ErrAlreadyExists) {
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error putting new status: %s", err))
}
return status, statusable, nil
return latest, apubStatus, nil
}
func (d *deref) dereferenceStatusable(ctx context.Context, tsport transport.Transport, remoteStatusID *url.URL) (ap.Statusable, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, remoteStatusID.Host); blocked || err != nil {
return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host)
// RefreshStatus: implements Dereferencer{}.RefreshStatus().
func (d *deref) RefreshStatus(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) (*gtsmodel.Status, ap.Statusable, error) {
// Check whether needs update.
if statusUpToDate(status) {
return status, nil, nil
}
b, err := tsport.Dereference(ctx, remoteStatusID)
// Parse the URI from status.
uri, err := url.Parse(status.URI)
if err != nil {
return nil, fmt.Errorf("dereferenceStatusable: error deferencing %s: %w", remoteStatusID.String(), err)
return nil, nil, fmt.Errorf("RefreshStatus: invalid status uri %q: %w", status.URI, err)
}
return ap.ResolveStatusable(ctx, b)
// Try to update + deref existing status model.
latest, apubStatus, err := d.enrichStatus(ctx,
requestUser,
uri,
status,
apubStatus,
)
if err != nil {
return nil, nil, err
}
// This status was updated, enqueue re-dereferencing the whole thread.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
d.dereferenceThread(ctx, requestUser, uri, latest, apubStatus)
})
return latest, apubStatus, nil
}
// populateStatusFields fetches all the information we temporarily pinned to an incoming
// federated status, back in the federating db's Create function.
//
// When a status comes in from the federation API, there are certain fields that
// haven't been dereferenced yet, because we needed to provide a snappy synchronous
// response to the caller. By the time it reaches this function though, it's being
// processed asynchronously, so we have all the time in the world to fetch the various
// bits and bobs that are attached to the status, and properly flesh it out, before we
// send the status to any timelines and notify people.
//
// Things to dereference and fetch here:
//
// 1. Media attachments.
// 2. Hashtags.
// 3. Emojis.
// 4. Mentions.
// 5. Replied-to-status.
//
// SIDE EFFECTS:
// This function will deference all of the above, insert them in the database as necessary,
// and attach them to the status. The status itself will not be added to the database yet,
// that's up the caller to do.
func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string, includeParent bool) error {
statusIRI, err := url.Parse(status.URI)
if err != nil {
return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err)
// RefreshStatusAsync: implements Dereferencer{}.RefreshStatusAsync().
func (d *deref) RefreshStatusAsync(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) {
// Check whether needs update.
if statusUpToDate(status) {
return
}
blocked, err := d.db.IsURIBlocked(ctx, statusIRI)
// Parse the URI from status.
uri, err := url.Parse(status.URI)
if err != nil {
return fmt.Errorf("populateStatusFields: error checking blocked status of %s: %s", statusIRI, err)
}
if blocked {
return fmt.Errorf("populateStatusFields: domain %s is blocked", statusIRI)
log.Errorf(ctx, "RefreshStatusAsync: invalid status uri %q: %v", status.URI, err)
return
}
// in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one
if status.ID == "" {
newID, err := id.NewULIDFromTime(status.CreatedAt)
// Enqueue a worker function to re-fetch this status async.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
latest, apubStatus, err := d.enrichStatus(ctx, requestUser, uri, status, apubStatus)
if err != nil {
return fmt.Errorf("populateStatusFields: error creating ulid for status: %s", err)
log.Errorf(ctx, "error enriching remote status: %v", err)
return
}
status.ID = newID
// This status was updated, re-dereference the whole thread.
d.dereferenceThread(ctx, requestUser, uri, latest, apubStatus)
})
}
// enrichStatus will enrich the given status, whether a new barebones model, or existing model from the database. It handles necessary dereferencing etc.
func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.URL, status *gtsmodel.Status, apubStatus ap.Statusable) (*gtsmodel.Status, ap.Statusable, error) {
// Pre-fetch a transport for requesting username, used by later dereferencing.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: couldn't create transport: %w", err)
}
// 1. Media attachments.
if err := d.populateStatusAttachments(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status attachments: %s", err)
// Check whether this account URI is a blocked domain / subdomain.
if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error checking blocked domain: %w", err)
} else if blocked {
return nil, nil, fmt.Errorf("enrichStatus: %s is blocked", uri.Host)
}
// 2. Hashtags
// TODO
var derefd bool
// 3. Emojis
if err := d.populateStatusEmojis(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status emojis: %s", err)
if apubStatus == nil {
// Dereference latest version of the status.
b, err := tsport.Dereference(ctx, uri)
if err != nil {
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichStatus: error deferencing %s: %w", uri, err)}
}
// Attempt to resolve ActivityPub status from data.
apubStatus, err = ap.ResolveStatusable(ctx, b)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error resolving statusable from data for account %s: %w", uri, err)
}
// Mark as deref'd.
derefd = true
}
// 4. Mentions
// TODO: do we need to handle removing empty mention objects and just using mention IDs slice?
if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err)
// Get the attributed-to status in order to fetch profile.
attributedTo, err := ap.ExtractAttributedTo(apubStatus)
if err != nil {
return nil, nil, errors.New("enrichStatus: attributedTo was empty")
}
// 5. Replied-to-status (only if requested)
if includeParent {
if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err)
// Ensure we have the author account of the status dereferenced (+ up-to-date).
if author, _, err := d.getAccountByURI(ctx, requestUser, attributedTo); err != nil {
if status.AccountID == "" {
// Provided status account is nil, i.e. this is a new status / author, so a deref fail is unrecoverable.
return nil, nil, fmt.Errorf("enrichStatus: failed to dereference status author %s: %w", uri, err)
}
} else if status.AccountID != "" && status.AccountID != author.ID {
// There already existed an account for this status author, but account ID changed. This shouldn't happen!
log.Warnf(ctx, "status author account ID changed: old=%s new=%s", status.AccountID, author.ID)
}
// By default we assume that apubStatus has been passed,
// indicating that the given status is already latest.
latestStatus := status
if derefd {
// ActivityPub model was recently dereferenced, so assume that passed status
// may contain out-of-date information, convert AP model to our GTS model.
latestStatus, err = d.typeConverter.ASStatusToStatus(ctx, apubStatus)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error converting statusable to gts model for status %s: %w", uri, err)
}
}
// Use existing status ID.
latestStatus.ID = status.ID
if latestStatus.ID == "" {
// Generate new status ID from the provided creation date.
latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: invalid created at date: %w", err)
}
}
// Carry-over values and set fetch time.
latestStatus.FetchedAt = time.Now()
latestStatus.Local = status.Local
// Ensure the status' mentions are populated, and pass in existing to check for changes.
if err := d.fetchStatusMentions(ctx, requestUser, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating mentions for status %s: %w", uri, err)
}
// TODO: populateStatusTags()
// Ensure the status' media attachments are populated, passing in existing to check for changes.
if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating attachments for status %s: %w", uri, err)
}
// Ensure the status' emoji attachments are populated, passing in existing to check for changes.
if err := d.fetchStatusEmojis(ctx, requestUser, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating emojis for status %s: %w", uri, err)
}
if status.CreatedAt.IsZero() {
// CreatedAt will be zero if no local copy was
// found in one of the GetStatusBy___() functions.
//
// This is new, put the status in the database.
err := d.state.DB.PutStatus(ctx, latestStatus)
if errors.Is(err, db.ErrAlreadyExists) {
// TODO: replace this quick fix with per-URI deref locks.
latestStatus, err = d.state.DB.GetStatusByURI(ctx, latestStatus.URI)
return latestStatus, nil, err
}
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error putting in database: %w", err)
}
} else {
// This is an existing status, update the model in the database.
if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error updating database: %w", err)
}
}
return latestStatus, apubStatus, nil
}
func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be created mention IDs.
status.MentionIDs = make([]string, len(status.Mentions))
for i := range status.Mentions {
mention := status.Mentions[i]
// Look for existing mention with target account URI first.
existing, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
if ok && existing.ID != "" {
status.Mentions[i] = existing
status.MentionIDs[i] = existing.ID
continue
}
// Ensure that mention account URI is parseable.
accountURI, err := url.Parse(mention.TargetAccountURI)
if err != nil {
log.Errorf(ctx, "invalid account uri %q: %v", mention.TargetAccountURI, err)
continue
}
// Ensure we have the account of the mention target dereferenced.
mention.TargetAccount, _, err = d.getAccountByURI(ctx, requestUser, accountURI)
if err != nil {
log.Errorf(ctx, "failed to dereference account %s: %v", accountURI, err)
continue
}
// Generate new ID according to status creation.
mention.ID, err = id.NewULIDFromTime(status.CreatedAt)
if err != nil {
log.Errorf(ctx, "invalid created at date: %v", err)
mention.ID = id.NewULID() // just use "now"
}
// Set known further mention details.
mention.CreatedAt = status.CreatedAt
mention.UpdatedAt = status.UpdatedAt
mention.OriginAccount = status.Account
mention.OriginAccountID = status.AccountID
mention.OriginAccountURI = status.AccountURI
mention.TargetAccountID = mention.TargetAccount.ID
mention.TargetAccountURI = mention.TargetAccount.URI
mention.TargetAccountURL = mention.TargetAccount.URL
mention.StatusID = status.ID
mention.Status = status
// Place the new mention into the database.
if err := d.state.DB.PutMention(ctx, mention); err != nil {
return fmt.Errorf("error putting mention in database: %w", err)
}
// Set the *new* mention and ID.
status.Mentions[i] = mention
status.MentionIDs[i] = mention.ID
}
for i := 0; i < len(status.MentionIDs); i++ {
if status.MentionIDs[i] == "" {
// This is a failed mention population, likely due
// to invalid incoming data / now-deleted accounts.
copy(status.Mentions[i:], status.Mentions[i+1:])
copy(status.MentionIDs[i:], status.MentionIDs[i+1:])
status.Mentions = status.Mentions[:len(status.Mentions)-1]
status.MentionIDs = status.MentionIDs[:len(status.MentionIDs)-1]
}
}
return nil
}
func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
// At this point, mentions should have the namestring and mentionedAccountURI set on them.
// We can use these to find the accounts.
func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be fetched attachment IDs.
status.AttachmentIDs = make([]string, len(status.Attachments))
mentionIDs := []string{}
newMentions := []*gtsmodel.Mention{}
for _, m := range status.Mentions {
if m.ID != "" {
// we've already populated this mention, since it has an ID
log.Debug(ctx, "mention already populated")
mentionIDs = append(mentionIDs, m.ID)
newMentions = append(newMentions, m)
for i := range status.Attachments {
placeholder := status.Attachments[i]
// Look for existing media attachment with remoet URL first.
existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL)
if ok && existing.ID != "" {
status.Attachments[i] = existing
status.AttachmentIDs[i] = existing.ID
continue
}
if m.TargetAccountURI == "" {
log.Debug(ctx, "target URI not set on mention")
continue
}
targetAccountURI, err := url.Parse(m.TargetAccountURI)
// Ensure a valid media attachment remote URL.
remoteURL, err := url.Parse(placeholder.RemoteURL)
if err != nil {
log.Debugf(ctx, "error parsing mentioned account uri %s: %s", m.TargetAccountURI, err)
log.Errorf(ctx, "invalid remote media url %q: %v", placeholder.RemoteURL, err)
continue
}
var targetAccount *gtsmodel.Account
errs := []string{}
// check if account is in the db already
if a, err := d.db.GetAccountByURI(ctx, targetAccountURI.String()); err != nil {
errs = append(errs, err.Error())
} else {
log.Debugf(ctx, "got target account %s with id %s through GetAccountByURI", targetAccountURI, a.ID)
targetAccount = a
}
if targetAccount == nil {
// we didn't find the account in our database already
// check if we can get the account remotely (dereference it)
if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI); err != nil {
errs = append(errs, err.Error())
} else {
log.Debugf(ctx, "got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)
targetAccount = a
}
}
if targetAccount == nil {
log.Debugf(ctx, "couldn't get target account %s: %s", m.TargetAccountURI, strings.Join(errs, " : "))
continue
}
mID, err := id.NewRandomULID()
if err != nil {
return fmt.Errorf("populateStatusMentions: error generating ulid: %s", err)
}
newMention := &gtsmodel.Mention{
ID: mID,
StatusID: status.ID,
Status: m.Status,
CreatedAt: status.CreatedAt,
UpdatedAt: status.UpdatedAt,
OriginAccountID: status.AccountID,
OriginAccountURI: status.AccountURI,
OriginAccount: status.Account,
TargetAccountID: targetAccount.ID,
TargetAccount: targetAccount,
NameString: m.NameString,
TargetAccountURI: targetAccount.URI,
TargetAccountURL: targetAccount.URL,
}
if err := d.db.PutMention(ctx, newMention); err != nil {
return fmt.Errorf("populateStatusMentions: error creating mention: %s", err)
}
mentionIDs = append(mentionIDs, newMention.ID)
newMentions = append(newMentions, newMention)
}
status.MentionIDs = mentionIDs
status.Mentions = newMentions
return nil
}
func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
// At this point we should know:
// * the media type of the file we're looking for (a.File.ContentType)
// * the file type (a.Type)
// * the remote URL (a.RemoteURL)
// This should be enough to dereference the piece of media.
attachmentIDs := []string{}
attachments := []*gtsmodel.MediaAttachment{}
for _, a := range status.Attachments {
a.AccountID = status.AccountID
a.StatusID = status.ID
processingMedia, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{
CreatedAt: &a.CreatedAt,
StatusID: &a.StatusID,
RemoteURL: &a.RemoteURL,
Description: &a.Description,
Blurhash: &a.Blurhash,
// Start pre-processing remote media at remote URL.
processing, err := d.mediaManager.PreProcessMedia(ctx, func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, remoteURL)
}, nil, status.AccountID, &media.AdditionalMediaInfo{
StatusID: &status.ID,
RemoteURL: &placeholder.RemoteURL,
Description: &placeholder.Description,
Blurhash: &placeholder.Blurhash,
})
if err != nil {
log.Errorf(ctx, "couldn't get remote media %s: %s", a.RemoteURL, err)
log.Errorf(ctx, "error processing attachment: %v", err)
continue
}
attachment, err := processingMedia.LoadAttachment(ctx)
// Force attachment loading *right now*.
media, err := processing.LoadAttachment(ctx)
if err != nil {
log.Errorf(ctx, "couldn't load remote attachment %s: %s", a.RemoteURL, err)
log.Errorf(ctx, "error loading attachment: %v", err)
continue
}
attachmentIDs = append(attachmentIDs, attachment.ID)
attachments = append(attachments, attachment)
// Set the *new* attachment and ID.
status.Attachments[i] = media
status.AttachmentIDs[i] = media.ID
}
status.AttachmentIDs = attachmentIDs
status.Attachments = attachments
for i := 0; i < len(status.AttachmentIDs); i++ {
if status.AttachmentIDs[i] == "" {
// This is a failed attachment population, this may
// be due to us not currently supporting a media type.
copy(status.Attachments[i:], status.Attachments[i+1:])
copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:])
status.Attachments = status.Attachments[:len(status.Attachments)-1]
status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1]
}
}
return nil
}
func (d *deref) populateStatusEmojis(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
emojis, err := d.populateEmojis(ctx, status.Emojis, requestingUsername)
func (d *deref) fetchStatusEmojis(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Fetch the full-fleshed-out emoji objects for our status.
emojis, err := d.populateEmojis(ctx, status.Emojis, requestUser)
if err != nil {
return err
return fmt.Errorf("failed to populate emojis: %w", err)
}
// Iterate over and get their IDs.
emojiIDs := make([]string, 0, len(emojis))
for _, e := range emojis {
emojiIDs = append(emojiIDs, e.ID)
}
// Set known emoji details.
status.Emojis = emojis
status.EmojiIDs = emojiIDs
return nil
}
func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
if status.InReplyToURI != "" && status.InReplyToID == "" {
statusURI, err := url.Parse(status.InReplyToURI)
if err != nil {
return err
}
replyToStatus, _, err := d.GetStatus(ctx, requestingUsername, statusURI, false, false)
if err != nil {
return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", status.InReplyToURI, err)
}
// we have the status
status.InReplyToID = replyToStatus.ID
status.InReplyTo = replyToStatus
status.InReplyToAccountID = replyToStatus.AccountID
status.InReplyToAccount = replyToStatus.Account
}
return nil
}

6
internal/federation/dereferencing/status_test.go
View file

@ -36,7 +36,7 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)
@ -76,7 +76,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)
@ -127,7 +127,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://turnip.farm/users/turniplover6969/statuses/70c53e54-3146-42d5-a630-83c8b6c7c042")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)

49
internal/federation/dereferencing/thread.go
View file

@ -35,34 +35,16 @@
// ancesters we are willing to follow before returning error.
const maxIter = 1000
// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo),
// and dereferences statusables in the conversation.
//
// This process involves working up and down the chain of replies, and parsing through the collections of IDs
// presented by remote instances as part of their replies collections, and will likely involve making several calls to
// multiple different hosts.
//
// This does not return error, as for robustness we do not want to error-out on a status because another further up / down has issues.
func (d *deref) DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
l := log.WithContext(ctx).
WithFields(kv.Fields{
{"username", username},
{"statusIRI", status.URI},
}...)
// Log function start
l.Trace("beginning")
// dereferenceThread will dereference statuses both above and below the given status in a thread, it returns no error and is intended to be called asychronously.
func (d *deref) dereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
// Ensure that ancestors have been fully dereferenced
if err := d.dereferenceStatusAncestors(ctx, username, status); err != nil {
l.Errorf("error dereferencing status ancestors: %v", err)
// we don't return error, we have deref'd as much as we can
log.Errorf(ctx, "error dereferencing status ancestors: %v", err)
}
// Ensure that descendants have been fully dereferenced
if err := d.dereferenceStatusDescendants(ctx, username, statusIRI, statusable); err != nil {
l.Errorf("error dereferencing status descendants: %v", err)
// we don't return error, we have deref'd as much as we can
log.Errorf(ctx, "error dereferencing status descendants: %v", err)
}
}
@ -103,7 +85,7 @@ func (d *deref) dereferenceStatusAncestors(ctx context.Context, username string,
}
// Fetch this status from the database
localStatus, err := d.db.GetStatusByID(ctx, id)
localStatus, err := d.state.DB.GetStatusByID(ctx, id)
if err != nil {
return fmt.Errorf("error fetching local status %q: %w", id, err)
}
@ -115,7 +97,10 @@ func (d *deref) dereferenceStatusAncestors(ctx context.Context, username string,
l.Tracef("following remote status ancestors: %s", status.InReplyToURI)
// Fetch the remote status found at this IRI
remoteStatus, _, err := d.GetStatus(ctx, username, replyIRI, false, false)
remoteStatus, _, err := d.getStatusByURI(ctx,
username,
replyIRI,
)
if err != nil {
return fmt.Errorf("error fetching remote status %q: %w", status.InReplyToURI, err)
}
@ -277,10 +262,15 @@ type frame struct {
continue itemLoop
}
// Dereference the remote status and store in the database
_, statusable, err := d.GetStatus(ctx, username, itemIRI, true, false)
// Dereference the remote status and store in the database.
_, statusable, err := d.getStatusByURI(ctx, username, itemIRI)
if err != nil {
l.Errorf("error dereferencing remote status %q: %s", itemIRI.String(), err)
l.Errorf("error dereferencing remote status %s: %v", itemIRI, err)
continue itemLoop
}
if statusable == nil {
// Already up-to-date.
continue itemLoop
}
@ -307,7 +297,10 @@ type frame struct {
}
// Dereference this next collection page by its IRI
collectionPage, err := d.DereferenceCollectionPage(ctx, username, pageNextIRI)
collectionPage, err := d.dereferenceCollectionPage(ctx,
username,
pageNextIRI,
)
if err != nil {
l.Errorf("error dereferencing remote collection page %q: %s", pageNextIRI.String(), err)
continue stackLoop

4
internal/federation/federatingactor_test.go
View file

@ -58,7 +58,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
activity, err := federator.FederatingActor().Send(ctx, testrig.URLMustParse(testAccount.OutboxURI), testActivity)
suite.NoError(err)
@ -103,7 +103,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
activity, err := federator.FederatingActor().Send(ctx, testrig.URLMustParse(testAccount.OutboxURI), testActivity)
suite.NoError(err)

2
internal/federation/federatingdb/create.go
View file

@ -205,6 +205,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
APObjectType: ap.ObjectNote,
APActivityType: ap.ActivityCreate,
APIri: id.GetIRI(),
APObjectModel: nil,
GTSModel: nil,
ReceivingAccount: receivingAccount,
})
@ -238,6 +239,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
f.state.Workers.EnqueueFederator(ctx, messages.FromFederator{
APObjectType: ap.ObjectNote,
APActivityType: ap.ActivityCreate,
APObjectModel: note,
GTSModel: status,
ReceivingAccount: receivingAccount,
})

4
internal/federation/federatingprotocol.go
View file

@ -210,9 +210,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
// We know the public key owner URI now, so we can
// dereference the remote account (or just get it
// from the db if we already have it).
requestingAccount, err := f.GetAccountByURI(
gtscontext.SetFastFail(ctx), username, publicKeyOwnerURI,
)
requestingAccount, _, err := f.GetAccountByURI(gtscontext.SetFastFail(ctx), username, publicKeyOwnerURI)
if err != nil {
if gtserror.StatusCode(err) == http.StatusGone {
// This is the same case as the http.StatusGone check above.

20
internal/federation/federatingprotocol_test.go
View file

@ -43,7 +43,7 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook1() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
// setup request
ctx := context.Background()
@ -73,7 +73,7 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook2() {
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
// setup request
ctx := context.Background()
@ -104,7 +104,7 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook3() {
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
// setup request
ctx := context.Background()
@ -137,7 +137,7 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() {
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// now setup module being tested, with the mock transport controller
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil)
// we need these headers for the request to be validated
@ -180,7 +180,7 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostGone() {
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// now setup module being tested, with the mock transport controller
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil)
// we need these headers for the request to be validated
@ -222,7 +222,7 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostGoneNoTombstoneYet
tc := testrig.NewTestTransportController(&suite.state, httpClient)
// now setup module being tested, with the mock transport controller
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil)
// we need these headers for the request to be validated
@ -258,7 +258,7 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostGoneNoTombstoneYet
func (suite *FederatingProtocolTestSuite) TestBlocked1() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
sendingAccount := suite.testAccounts["remote_account_1"]
inboxAccount := suite.testAccounts["local_account_1"]
@ -280,7 +280,7 @@ func (suite *FederatingProtocolTestSuite) TestBlocked1() {
func (suite *FederatingProtocolTestSuite) TestBlocked2() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
sendingAccount := suite.testAccounts["remote_account_1"]
inboxAccount := suite.testAccounts["local_account_1"]
@ -313,7 +313,7 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() {
func (suite *FederatingProtocolTestSuite) TestBlocked3() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
sendingAccount := suite.testAccounts["remote_account_1"]
inboxAccount := suite.testAccounts["local_account_1"]
@ -349,7 +349,7 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() {
func (suite *FederatingProtocolTestSuite) TestBlocked4() {
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(&suite.state, httpClient)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
federator := federation.NewFederator(&suite.state, testrig.NewTestFederatingDB(&suite.state), tc, suite.tc, testrig.NewTestMediaManager(&suite.state))
sendingAccount := suite.testAccounts["remote_account_1"]
inboxAccount := suite.testAccounts["local_account_1"]

7
internal/federation/federator.go
View file

@ -27,6 +27,7 @@
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -67,12 +68,12 @@ type federator struct {
}
// NewFederator returns a new federator
func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaManager media.Manager) Federator {
dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaManager)
func NewFederator(state *state.State, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaManager media.Manager) Federator {
dereferencer := dereferencing.NewDereferencer(state, typeConverter, transportController, mediaManager)
clock := &Clock{}
f := &federator{
db: db,
db: state.DB,
federatingDB: federatingDB,
clock: &Clock{},
typeConverter: typeConverter,

105
internal/gtsmodel/status.go
View file

@ -28,6 +28,7 @@ type Status struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
FetchedAt time.Time `validate:"required_with=!Local" bun:"type:timestamptz,nullzero"` // when was item (remote) last fetched.
PinnedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // Status was pinned by owning account at this time.
URI string `validate:"required,url" bun:",unique,nullzero,notnull"` // activitypub URI of this status
URL string `validate:"url" bun:",nullzero"` // web url for viewing this status
@ -87,24 +88,43 @@ func (s *Status) GetBoostOfAccountID() string {
return s.BoostOfAccountID
}
func (s *Status) GetAttachmentByID(id string) (*MediaAttachment, bool) {
for _, media := range s.Attachments {
if media == nil {
log.Warnf(nil, "nil attachment in slice for status %s", s.URI)
continue
}
if media.ID == id {
return media, true
}
}
return nil, false
}
func (s *Status) GetAttachmentByRemoteURL(url string) (*MediaAttachment, bool) {
for _, media := range s.Attachments {
if media == nil {
log.Warnf(nil, "nil attachment in slice for status %s", s.URI)
continue
}
if media.RemoteURL == url {
return media, true
}
}
return nil, false
}
// AttachmentsPopulated returns whether media attachments are populated according to current AttachmentIDs.
func (s *Status) AttachmentsPopulated() bool {
if len(s.AttachmentIDs) != len(s.Attachments) {
// this is the quickest indicator.
return false
}
// Attachments must be in same order.
for i, id := range s.AttachmentIDs {
if s.Attachments[i] == nil {
log.Warnf(nil, "nil attachment in slice for status %s", s.URI)
continue
}
if s.Attachments[i].ID != id {
for _, id := range s.AttachmentIDs {
if _, ok := s.GetAttachmentByID(id); !ok {
return false
}
}
return true
}
@ -129,24 +149,43 @@ func (s *Status) TagsPopulated() bool {
return true
}
func (s *Status) GetMentionByID(id string) (*Mention, bool) {
for _, mention := range s.Mentions {
if mention == nil {
log.Warnf(nil, "nil mention in slice for status %s", s.URI)
continue
}
if mention.ID == id {
return mention, true
}
}
return nil, false
}
func (s *Status) GetMentionByTargetURI(uri string) (*Mention, bool) {
for _, mention := range s.Mentions {
if mention == nil {
log.Warnf(nil, "nil mention in slice for status %s", s.URI)
continue
}
if mention.TargetAccountURI == uri {
return mention, true
}
}
return nil, false
}
// MentionsPopulated returns whether mentions are populated according to current MentionIDs.
func (s *Status) MentionsPopulated() bool {
if len(s.MentionIDs) != len(s.Mentions) {
// this is the quickest indicator.
return false
}
// Mentions must be in same order.
for i, id := range s.MentionIDs {
if s.Mentions[i] == nil {
log.Warnf(nil, "nil mention in slice for status %s", s.URI)
continue
}
if s.Mentions[i].ID != id {
for _, id := range s.MentionIDs {
if _, ok := s.GetMentionByID(id); !ok {
return false
}
}
return true
}
@ -171,6 +210,36 @@ func (s *Status) EmojisPopulated() bool {
return true
}
// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date
// according to emoji attachments of the passed status, by comparing their emoji URIs. We don't
// use IDs as this is used to determine whether there are new emojis to fetch.
func (s *Status) EmojisUpToDate(other *Status) bool {
if len(s.Emojis) != len(other.Emojis) {
// this is the quickest indicator.
return false
}
// Emojis must be in same order.
for i := range s.Emojis {
if s.Emojis[i] == nil {
log.Warnf(nil, "nil emoji in slice for status %s", s.URI)
return false
}
if other.Emojis[i] == nil {
log.Warnf(nil, "nil emoji in slice for status %s", other.URI)
return false
}
if s.Emojis[i].URI != other.Emojis[i].URI {
// Emoji URI has changed, not up-to-date!
return false
}
}
return true
}
// MentionsAccount returns whether status mentions the given account ID.
func (s *Status) MentionsAccount(id string) bool {
for _, mention := range s.Mentions {

1
internal/media/processingemoji.go
View file

@ -123,7 +123,6 @@ func (p *ProcessingEmoji) load(ctx context.Context) (*gtsmodel.Emoji, bool, erro
if p.refresh {
columns := []string{
"updated_at",
"image_remote_url",
"image_static_remote_url",
"image_url",

6
internal/media/processingmedia.go
View file

@ -245,8 +245,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
info.Extension,
)
p.media.File.ContentType = info.MIME.Value
cached := true
p.media.Cached = &cached
p.media.Cached = func() *bool {
ok := true
return &ok
}()
return nil
}

67
internal/processing/account/get.go
View file

@ -28,16 +28,17 @@
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
// Get processes the given request for account information.
func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Account, gtserror.WithCode) {
targetAccount, err := p.state.DB.GetAccountByID(ctx, targetAccountID)
if err != nil {
if err == db.ErrNoEntries {
if errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorNotFound(errors.New("account not found"))
}
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %s", err))
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %w", err))
}
return p.getFor(ctx, requestingAccount, targetAccount)
@ -47,10 +48,10 @@ func (p *Processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account
func (p *Processor) GetLocalByUsername(ctx context.Context, requestingAccount *gtsmodel.Account, username string) (*apimodel.Account, gtserror.WithCode) {
targetAccount, err := p.state.DB.GetAccountByUsernameDomain(ctx, username, "")
if err != nil {
if err == db.ErrNoEntries {
if errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.NewErrorNotFound(errors.New("account not found"))
}
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %s", err))
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error: %w", err))
}
return p.getFor(ctx, requestingAccount, targetAccount)
@ -60,48 +61,50 @@ func (p *Processor) GetLocalByUsername(ctx context.Context, requestingAccount *g
func (p *Processor) GetCustomCSSForUsername(ctx context.Context, username string) (string, gtserror.WithCode) {
customCSS, err := p.state.DB.GetAccountCustomCSSByUsername(ctx, username)
if err != nil {
if err == db.ErrNoEntries {
if errors.Is(err, db.ErrNoEntries) {
return "", gtserror.NewErrorNotFound(errors.New("account not found"))
}
return "", gtserror.NewErrorInternalError(fmt.Errorf("db error: %s", err))
return "", gtserror.NewErrorInternalError(fmt.Errorf("db error: %w", err))
}
return customCSS, nil
}
func (p *Processor) getFor(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccount *gtsmodel.Account) (*apimodel.Account, gtserror.WithCode) {
var blocked bool
var err error
if requestingAccount != nil {
blocked, err = p.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, targetAccount.ID)
blocked, err := p.state.DB.IsEitherBlocked(ctx, requestingAccount.ID, targetAccount.ID)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking account block: %s", err))
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking account block: %w", err))
}
if blocked {
apiAccount, err := p.tc.AccountToAPIAccountBlocked(ctx, targetAccount)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting account: %w", err))
}
return apiAccount, nil
}
}
if targetAccount.Domain != "" {
targetAccountURI, err := url.Parse(targetAccount.URI)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %w", targetAccount.URI, err))
}
// Perform a last-minute fetch of target account to ensure remote account header / avatar is cached.
latest, _, err := p.federator.GetAccountByURI(gtscontext.SetFastFail(ctx), requestingAccount.Username, targetAccountURI)
if err != nil {
log.Errorf(ctx, "error fetching latest target account: %v", err)
} else {
// Use latest account model.
targetAccount = latest
}
}
var apiAccount *apimodel.Account
if blocked {
apiAccount, err = p.tc.AccountToAPIAccountBlocked(ctx, targetAccount)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting account: %s", err))
}
return apiAccount, nil
}
// last-minute check to make sure we have remote account header/avi cached
if targetAccount.Domain != "" {
targetAccountURI, err := url.Parse(targetAccount.URI)
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err))
}
a, err := p.federator.GetAccountByURI(
gtscontext.SetFastFail(ctx), requestingAccount.Username, targetAccountURI,
)
if err == nil {
targetAccount = a
}
}
if requestingAccount != nil && targetAccount.ID == requestingAccount.ID {
apiAccount, err = p.tc.AccountToAPIAccountSensitive(ctx, targetAccount)
@ -109,7 +112,7 @@ func (p *Processor) getFor(ctx context.Context, requestingAccount *gtsmodel.Acco
apiAccount, err = p.tc.AccountToAPIAccountPublic(ctx, targetAccount)
}
if err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting account: %s", err))
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting account: %w", err))
}
return apiAccount, nil

4
internal/processing/admin/emoji.go
View file

@ -385,7 +385,7 @@ func (p *Processor) emojiUpdateDisable(ctx context.Context, emoji *gtsmodel.Emoj
emojiDisabled := true
emoji.Disabled = &emojiDisabled
updatedEmoji, err := p.state.DB.UpdateEmoji(ctx, emoji, "updated_at", "disabled")
updatedEmoji, err := p.state.DB.UpdateEmoji(ctx, emoji, "disabled")
if err != nil {
err = fmt.Errorf("emojiUpdateDisable: error updating emoji %s: %s", emoji.ID, err)
return nil, gtserror.NewErrorInternalError(err)
@ -434,7 +434,7 @@ func (p *Processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji
if !updateImage {
// only updating fields, we only need
// to do a database update for this
columns := []string{"updated_at"}
var columns []string
if updateCategoryID {
emoji.CategoryID = updatedCategoryID

2
internal/processing/fedi/common.go
View file

@ -40,7 +40,7 @@ func (p *Processor) authenticate(ctx context.Context, requestedUsername string)
return
}
if requestingAccount, err = p.federator.GetAccountByURI(gtscontext.SetFastFail(ctx), requestedUsername, requestingAccountURI); err != nil {
if requestingAccount, _, err = p.federator.GetAccountByURI(gtscontext.SetFastFail(ctx), requestedUsername, requestingAccountURI); err != nil {
errWithCode = gtserror.NewErrorUnauthorized(err)
return
}

4
internal/processing/fedi/user.go
View file

@ -55,9 +55,7 @@ func (p *Processor) UserGet(ctx context.Context, requestedUsername string, reque
// if we're not already handshaking/dereferencing a remote account, dereference it now
if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
requestingAccount, err := p.federator.GetAccountByURI(
gtscontext.SetFastFail(ctx), requestedUsername, requestingAccountURI,
)
requestingAccount, _, err := p.federator.GetAccountByURI(gtscontext.SetFastFail(ctx), requestedUsername, requestingAccountURI)
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

59
internal/processing/fromfederator.go
View file

@ -110,17 +110,30 @@ func (p *Processor) ProcessFromFederator(ctx context.Context, federatorMsg messa
func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federatorMsg messages.FromFederator) error {
// check for either an IRI that we still need to dereference, OR an already dereferenced
// and converted status pinned to the message.
var status *gtsmodel.Status
var (
status *gtsmodel.Status
err error
)
if federatorMsg.GTSModel != nil {
// there's a gts model already pinned to the message, it should be a status
var ok bool
// there's a gts model already pinned to the message, it should be a status
if status, ok = federatorMsg.GTSModel.(*gtsmodel.Status); !ok {
return errors.New("ProcessFromFederator: note was not parseable as *gtsmodel.Status")
}
var err error
status, err = p.federator.EnrichRemoteStatus(ctx, federatorMsg.ReceivingAccount.Username, status, true)
// Since this was a create originating AP object
// statusable may have been set on message (no problem if not).
statusable, _ := federatorMsg.APObjectModel.(ap.Statusable)
// Call refresh on status to deref if necessary etc.
status, _, err = p.federator.RefreshStatus(ctx,
federatorMsg.ReceivingAccount.Username,
status,
statusable,
false,
)
if err != nil {
return err
}
@ -129,38 +142,29 @@ func (p *Processor) processCreateStatusFromFederator(ctx context.Context, federa
if federatorMsg.APIri == nil {
return errors.New("ProcessFromFederator: status was not pinned to federatorMsg, and neither was an IRI for us to dereference")
}
var err error
status, _, err = p.federator.GetStatus(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri, false, false)
status, _, err = p.federator.GetStatusByURI(ctx, federatorMsg.ReceivingAccount.Username, federatorMsg.APIri)
if err != nil {
return err
}
}
// make sure the account is pinned
if status.Account == nil {
a, err := p.state.DB.GetAccountByID(ctx, status.AccountID)
if err != nil {
return err
}
status.Account = a
}
// Get the remote account to make sure the avi and header are cached.
if status.Account.Domain != "" {
remoteAccountID, err := url.Parse(status.Account.URI)
if status.Account == nil || status.Account.IsRemote() {
// Either no account attached yet, or a remote account.
// Both situations we need to parse account URI to fetch it.
remoteAccURI, err := url.Parse(status.AccountURI)
if err != nil {
return err
}
a, err := p.federator.GetAccountByURI(ctx,
// Ensure that account for this status has been deref'd.
status.Account, _, err = p.federator.GetAccountByURI(ctx,
federatorMsg.ReceivingAccount.Username,
remoteAccountID,
remoteAccURI,
)
if err != nil {
return err
}
status.Account = a
}
if err := p.timelineAndNotifyStatus(ctx, status); err != nil {
@ -193,7 +197,7 @@ func (p *Processor) processCreateFaveFromFederator(ctx context.Context, federato
return err
}
a, err := p.federator.GetAccountByURI(ctx,
a, _, err := p.federator.GetAccountByURI(ctx,
federatorMsg.ReceivingAccount.Username,
remoteAccountID,
)
@ -234,7 +238,7 @@ func (p *Processor) processCreateFollowRequestFromFederator(ctx context.Context,
return err
}
a, err := p.federator.GetAccountByURI(ctx,
a, _, err := p.federator.GetAccountByURI(ctx,
federatorMsg.ReceivingAccount.Username,
remoteAccountID,
)
@ -294,7 +298,7 @@ func (p *Processor) processCreateAnnounceFromFederator(ctx context.Context, fede
return err
}
a, err := p.federator.GetAccountByURI(ctx,
a, _, err := p.federator.GetAccountByURI(ctx,
federatorMsg.ReceivingAccount.Username,
remoteAccountID,
)
@ -376,11 +380,12 @@ func (p *Processor) processUpdateAccountFromFederator(ctx context.Context, feder
}
// Call RefreshAccount to fetch up-to-date bio, avatar, header, etc.
updatedAccount, err := p.federator.RefreshAccount(
updatedAccount, _, err := p.federator.RefreshAccount(
ctx,
federatorMsg.ReceivingAccount.Username,
incomingAccountable,
incomingAccount,
incomingAccountable,
true,
)
if err != nil {
return fmt.Errorf("error enriching updated account from federator: %s", err)

13
internal/processing/fromfederator_test.go
View file

@ -142,15 +142,10 @@ func (suite *FromFederatorTestSuite) TestProcessReplyMention() {
suite.NoError(err)
// 2. a notification should exist for the mention
where := []db.Where{
{
Key: "status_id",
Value: replyingStatus.ID,
},
}
notif := &gtsmodel.Notification{}
err = suite.db.GetWhere(context.Background(), where, notif)
var notif gtsmodel.Notification
err = suite.db.GetWhere(context.Background(), []db.Where{
{Key: "status_id", Value: replyingStatus.ID},
}, &notif)
suite.NoError(err)
suite.Equal(gtsmodel.NotificationMention, notif.NotificationType)
suite.Equal(replyingStatus.InReplyToAccountID, notif.TargetAccountID)

2
internal/processing/processor.go
View file

@ -131,7 +131,7 @@ func NewProcessor(
processor.fedi = fedi.New(state, tc, federator, filter)
processor.media = media.New(state, tc, mediaManager, federator.TransportController())
processor.report = report.New(state, tc)
processor.status = status.New(state, tc, filter, parseMentionFunc)
processor.status = status.New(state, federator, tc, filter, parseMentionFunc)
processor.stream = stream.New(state, oauthServer)
processor.user = user.New(state, emailSender)

2
internal/processing/processor_test.go
View file

@ -100,6 +100,8 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.state.Storage = suite.storage
suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")
suite.httpClient.TestRemotePeople = testrig.NewTestFediPeople()
suite.httpClient.TestRemoteStatuses = testrig.NewTestFediStatuses()
suite.transportController = testrig.NewTestTransportController(&suite.state, suite.httpClient)
suite.mediaManager = testrig.NewTestMediaManager(&suite.state)

19
internal/processing/search.go
View file

@ -226,17 +226,8 @@ func (p *Processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *a
}
func (p *Processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL) (*gtsmodel.Status, error) {
status, statusable, err := p.federator.GetStatus(gtscontext.SetFastFail(ctx), authed.Account.Username, uri, true, true)
if err != nil {
return nil, err
}
if !*status.Local && statusable != nil {
// Attempt to dereference the status thread while we are here
p.federator.DereferenceThread(gtscontext.SetFastFail(ctx), authed.Account.Username, uri, status, statusable)
}
return status, nil
status, _, err := p.federator.GetStatusByURI(gtscontext.SetFastFail(ctx), authed.Account.Username, uri)
return status, err
}
func (p *Processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Account, error) {
@ -267,11 +258,12 @@ func (p *Processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
return account, nil
}
return p.federator.GetAccountByURI(
account, _, err := p.federator.GetAccountByURI(
gtscontext.SetFastFail(ctx),
authed.Account.Username,
uri,
)
return account, err
}
func (p *Processor) searchAccountByUsernameDomain(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
@ -294,9 +286,10 @@ func (p *Processor) searchAccountByUsernameDomain(ctx context.Context, authed *o
return account, nil
}
return p.federator.GetAccountByUsernameDomain(
account, _, err := p.federator.GetAccountByUsernameDomain(
gtscontext.SetFastFail(ctx),
authed.Account.Username,
username, domain,
)
return account, err
}

10
internal/processing/status/common.go
View file

@ -43,6 +43,16 @@ func (p *Processor) getVisibleStatus(ctx context.Context, requestingAccount *gts
return nil, gtserror.NewErrorNotFound(err)
}
if requestingAccount != nil {
// Ensure the status is up-to-date.
p.federator.RefreshStatusAsync(ctx,
requestingAccount.Username,
targetStatus,
nil,
false,
)
}
visible, err := p.filter.StatusVisible(ctx, requestingAccount, targetStatus)
if err != nil {
err = fmt.Errorf("getVisibleStatus: error seeing if status %s is visible: %w", targetStatus.ID, err)

5
internal/processing/status/status.go
View file

@ -18,6 +18,7 @@
package status
import (
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/text"
@ -27,6 +28,7 @@
type Processor struct {
state *state.State
federator federation.Federator
tc typeutils.TypeConverter
filter *visibility.Filter
formatter text.Formatter
@ -34,9 +36,10 @@ type Processor struct {
}
// New returns a new status processor.
func New(state *state.State, tc typeutils.TypeConverter, filter *visibility.Filter, parseMention gtsmodel.ParseMentionFunc) Processor {
func New(state *state.State, federator federation.Federator, tc typeutils.TypeConverter, filter *visibility.Filter, parseMention gtsmodel.ParseMentionFunc) Processor {
return Processor{
state: state,
federator: federator,
tc: tc,
filter: filter,
formatter: text.NewFormatter(state.DB),

2
internal/processing/status/status_test.go
View file

@ -88,7 +88,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.federator = testrig.NewTestFederator(&suite.state, suite.tc, suite.mediaManager)
filter := visibility.NewFilter(&suite.state)
suite.status = status.New(&suite.state, suite.typeConverter, filter, processing.GetParseMentionFunc(suite.db, suite.federator))
suite.status = status.New(&suite.state, suite.federator, suite.typeConverter, filter, processing.GetParseMentionFunc(suite.db, suite.federator))
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")

2
internal/processing/util.go
View file

@ -57,7 +57,7 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.
requestingUsername = originAccount.Username
}
remoteAccount, err := federator.GetAccountByUsernameDomain(
remoteAccount, _, err := federator.GetAccountByUsernameDomain(
gtscontext.SetFastFail(ctx),
requestingUsername,
username,

2
testrig/federator.go
View file

@ -26,5 +26,5 @@
// NewTestFederator returns a federator with the given database and (mock!!) transport controller.
func NewTestFederator(state *state.State, tc transport.Controller, mediaManager media.Manager) federation.Federator {
return federation.NewFederator(state.DB, NewTestFederatingDB(state), tc, NewTestTypeConverter(state.DB), mediaManager)
return federation.NewFederator(state, NewTestFederatingDB(state), tc, NewTestTypeConverter(state.DB), mediaManager)
}

20
testrig/testmodels.go
View file

@ -2482,6 +2482,26 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote {
),
},
),
"http://fossbros-anonymous.io/users/foss_satan/statuses/106221634728637552": NewAPNote(
URLMustParse("http://fossbros-anonymous.io/users/foss_satan/statuses/106221634728637552"),
URLMustParse("http://fossbros-anonymous.io/@foss_satan/106221634728637552"),
TimeMustParse("2022-07-13T12:13:12+02:00"),
`<p><span class="h-card"><a href="http://localhost:8080/@the_mighty_zork" class="u-url mention">@<span>the_mighty_zork</span></a></span> nice there it is:</p><p><a href="http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/activity" rel="nofollow noopener noreferrer" target="_blank"><span class="invisible">https://</span><span class="ellipsis">social.pixie.town/users/f0x/st</span><span class="invisible">atuses/106221628567855262/activity</span></a></p>`,
"",
URLMustParse("http://fossbros-anonymous.io/users/foss_satan"),
[]*url.URL{
URLMustParse(pub.PublicActivityPubIRI),
},
[]*url.URL{},
false,
[]vocab.ActivityStreamsMention{
newAPMention(
URLMustParse("http://localhost:8080/users/the_mighty_zork"),
"@the_mighty_zork@localhost:8080",
),
},
nil,
),
}
}