From a83e4fb7cb6ae09424314ef73abd7b1fe3ab1241 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Sat, 25 Jan 2025 22:13:25 -0800 Subject: [PATCH] Implement backfilling statuses thru scheduled_at --- docs/api/swagger.yaml | 10 ++- internal/api/client/statuses/statuscreate.go | 16 ++-- internal/api/model/status.go | 15 +++- internal/gtsmodel/status.go | 9 +- internal/processing/status/create.go | 86 +++++++++++++++++++- internal/processing/workers/fromclientapi.go | 25 ++++-- 6 files changed, 132 insertions(+), 29 deletions(-) diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 5165f8c9e..ec2d52f0a 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -10343,10 +10343,14 @@ paths: x-go-name: Federated - description: |- ISO 8601 Datetime at which to schedule a status. - Providing this parameter will cause ScheduledStatus to be returned instead of Status. - Must be at least 5 minutes in the future. - This feature isn't implemented yet; attemping to set it will return 501 Not Implemented. + Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. + Must be at least 5 minutes in the future. + This feature isn't implemented yet. + + Providing this parameter with a *past* time will cause the status to be backdated, + and will not push it to the user's followers. This is intended for importing old statuses. + format: date-time in: formData name: scheduled_at type: string diff --git a/internal/api/client/statuses/statuscreate.go b/internal/api/client/statuses/statuscreate.go index c83cdbad7..2cd3a807c 100644 --- a/internal/api/client/statuses/statuscreate.go +++ b/internal/api/client/statuses/statuscreate.go @@ -175,11 +175,15 @@ // x-go-name: ScheduledAt // description: |- // ISO 8601 Datetime at which to schedule a status. -// Providing this parameter will cause ScheduledStatus to be returned instead of Status. -// Must be at least 5 minutes in the future. // -// This feature isn't implemented yet; attemping to set it will return 501 Not Implemented. +// Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. +// Must be at least 5 minutes in the future. +// This feature isn't implemented yet. +// +// Providing this parameter with a *past* time will cause the status to be backdated, +// and will not push it to the user's followers. This is intended for importing old statuses. // type: string +// format: date-time // in: formData // - // name: language @@ -380,12 +384,6 @@ func parseStatusCreateForm(c *gin.Context) (*apimodel.StatusCreateRequest, gtser return nil, gtserror.NewErrorNotAcceptable(errors.New(text), text) } - // Check not scheduled status. - if form.ScheduledAt != "" { - const text = "scheduled_at is not yet implemented" - return nil, gtserror.NewErrorNotImplemented(errors.New(text), text) - } - // Check if the deprecated "federated" field was // set in lieu of "local_only", and use it if so. if form.LocalOnly == nil && form.Federated != nil { // nolint:staticcheck diff --git a/internal/api/model/status.go b/internal/api/model/status.go index ea9fbaa35..2ee3123e6 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -17,7 +17,11 @@ package model -import "github.com/superseriousbusiness/gotosocial/internal/language" +import ( + "time" + + "github.com/superseriousbusiness/gotosocial/internal/language" +) // Status models a status or post. // @@ -231,9 +235,14 @@ type StatusCreateRequest struct { Federated *bool `form:"federated" json:"federated"` // ISO 8601 Datetime at which to schedule a status. - // Providing this parameter will cause ScheduledStatus to be returned instead of Status. + // + // Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. // Must be at least 5 minutes in the future. - ScheduledAt string `form:"scheduled_at" json:"scheduled_at"` + // This feature isn't implemented yet. + // + // Providing this parameter with a *past* time will cause the status to be backdated, + // and will not push it to the user's followers. This is intended for importing old statuses. + ScheduledAt *time.Time `form:"scheduled_at" json:"scheduled_at"` // ISO 639 language code for this status. Language string `form:"language" json:"language"` diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index d28898ed1..e170e7464 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -86,7 +86,7 @@ func (s *Status) GetAccountID() string { return s.AccountID } -// GetBoostID implements timeline.Timelineable{}. +// GetBoostOfID implements timeline.Timelineable{}. func (s *Status) GetBoostOfID() string { return s.BoostOfID } @@ -171,7 +171,7 @@ func (s *Status) EditsPopulated() bool { return true } -// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date +// EmojisUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. func (s *Status) EmojisUpToDate(other *Status) bool { @@ -386,3 +386,8 @@ type Content struct { Content string ContentMap map[string]string } + +// BackfillStatus is a wrapper for creating a status without pushing notifications to followers. +type BackfillStatus struct { + *Status +} diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index b77d0af9c..249f20710 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -19,10 +19,13 @@ import ( "context" + "errors" "time" "github.com/superseriousbusiness/gotosocial/internal/ap" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" @@ -92,11 +95,35 @@ func (p *Processor) Create( // Get current time. now := time.Now() + // Default to current time as creation time. + createdAt := now + + // Handle backfilled/scheduled statuses. + backfill := false + if form.ScheduledAt != nil { + scheduledAt := *form.ScheduledAt + + // Statuses may only be scheduled a minimum time into the future. + if now.Before(scheduledAt) { + const errText = "scheduled statuses are not yet supported" + err := gtserror.New(errText) + return nil, gtserror.NewErrorBadRequest(err, errText) + } + + // If not scheduled into the future, this status is being backfilled. + backfill = true + createdAt = scheduledAt + var err error + if statusID, err = p.backfilledStatusID(ctx, createdAt); err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } + status := >smodel.Status{ ID: statusID, URI: accountURIs.StatusesURI + "/" + statusID, URL: accountURIs.StatusesURL + "/" + statusID, - CreatedAt: now, + CreatedAt: createdAt, Local: util.Ptr(true), Account: requester, AccountID: requester.ID, @@ -139,6 +166,7 @@ func (p *Processor) Create( requester, status, form.InReplyToID, + backfill, ); errWithCode != nil { return nil, errWithCode } @@ -165,11 +193,17 @@ func (p *Processor) Create( } if form.Poll != nil { + if backfill { + const errText = "posts with polls can't be backfilled" + err := gtserror.New(errText) + return nil, gtserror.NewErrorBadRequest(err, errText) + } + // Process poll, inserting into database. poll, errWithCode := p.processPoll(ctx, statusID, form.Poll, - now, + createdAt, ) if errWithCode != nil { return nil, errWithCode @@ -200,10 +234,14 @@ func (p *Processor) Create( } // Send it to the client API worker for async side-effects. + var model any = status + if backfill { + model = >smodel.BackfillStatus{Status: status} + } p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityCreate, - GTSModel: status, + GTSModel: model, Origin: requester, }) @@ -227,7 +265,40 @@ func (p *Processor) Create( return p.c.GetAPIStatus(ctx, requester, status) } -func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status, inReplyToID string) gtserror.WithCode { +// backfilledStatusID tries to find an unused ULID for a backfilled status. +func (p *Processor) backfilledStatusID(ctx context.Context, createdAt time.Time) (string, error) { + // backfilledStatusIDRetries should be more than enough attempts. + const backfilledStatusIDRetries = 100 + + for try := 0; try < backfilledStatusIDRetries; try++ { + var err error + + // Generate a ULID based on the backfilled status's original creation time. + statusID := id.NewULIDFromTime(createdAt) + + // Check for an existing status with that ID. + _, err = p.state.DB.GetStatusByID(gtscontext.SetBarebones(ctx), statusID) + if errors.Is(err, db.ErrNoEntries) { + // We found an unused one. + return statusID, nil + } else if err != nil { + err := gtserror.Newf("DB error checking if a status ID was in use: %w", err) + return "", err + } + // That status ID is in use. Try again. + } + + err := gtserror.Newf("failed to find an unused ID after %d tries", backfilledStatusIDRetries) + return "", err +} + +func (p *Processor) processInReplyTo( + ctx context.Context, + requester *gtsmodel.Account, + status *gtsmodel.Status, + inReplyToID string, + backfill bool, +) gtserror.WithCode { if inReplyToID == "" { // Not a reply. // Nothing to do. @@ -269,6 +340,13 @@ func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Ac return gtserror.NewErrorForbidden(err, errText) } + // When backfilling, only self-replies are allowed. + if backfill && requester.ID != inReplyTo.AccountID { + const errText = "replies to others can't be backfilled" + err := gtserror.New(errText) + return gtserror.NewErrorBadRequest(err, errText) + } + // Derive pendingApproval status. var pendingApproval bool switch { diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go index c5dfc157d..a208d97b0 100644 --- a/internal/processing/workers/fromclientapi.go +++ b/internal/processing/workers/fromclientapi.go @@ -260,9 +260,16 @@ func (p *clientAPI) CreateUser(ctx context.Context, cMsg *messages.FromClientAPI } func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientAPI) error { - status, ok := cMsg.GTSModel.(*gtsmodel.Status) - if !ok { - return gtserror.Newf("%T not parseable as *gtsmodel.Status", cMsg.GTSModel) + var status *gtsmodel.Status + backfill := false + if backfillStatus, ok := cMsg.GTSModel.(*gtsmodel.BackfillStatus); ok { + status = backfillStatus.Status + backfill = true + } else { + status, ok = cMsg.GTSModel.(*gtsmodel.Status) + if !ok { + return gtserror.Newf("%T not parseable as *gtsmodel.Status or *gtsmodel.BackfillStatus", cMsg.GTSModel) + } } // If pending approval is true then status must @@ -344,12 +351,14 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA log.Errorf(ctx, "error updating account stats: %v", err) } - if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { - log.Errorf(ctx, "error timelining and notifying status: %v", err) - } + if !backfill { + if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { + log.Errorf(ctx, "error timelining and notifying status: %v", err) + } - if err := p.federate.CreateStatus(ctx, status); err != nil { - log.Errorf(ctx, "error federating status: %v", err) + if err := p.federate.CreateStatus(ctx, status); err != nil { + log.Errorf(ctx, "error federating status: %v", err) + } } if status.InReplyToID != "" {