Implement backfilling statuses thru scheduled_at

docs/api/swagger.yaml

@@ -10343,10 +10343,14 @@ paths:
                   x-go-name: Federated
                 - description: |-
                     ISO 8601 Datetime at which to schedule a status.
-                    Providing this parameter will cause ScheduledStatus to be returned instead of Status.
-                    Must be at least 5 minutes in the future.
 
-                    This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+                    Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
+                    Must be at least 5 minutes in the future.
+                    This feature isn't implemented yet.
+
+                    Providing this parameter with a *past* time will cause the status to be backdated,
+                    and will not push it to the user's followers. This is intended for importing old statuses.
+                  format: date-time
                   in: formData
                   name: scheduled_at
                   type: string

internal/api/client/statuses/statuscreate.go

@@ -175,11 +175,15 @@
 //		x-go-name: ScheduledAt
 //		description: |-
 //			ISO 8601 Datetime at which to schedule a status.
-//			Providing this parameter will cause ScheduledStatus to be returned instead of Status.
-//			Must be at least 5 minutes in the future.
 //
-//			This feature isn't implemented yet; attemping to set it will return 501 Not Implemented.
+//			Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
+//			Must be at least 5 minutes in the future.
+//			This feature isn't implemented yet.
+//
+//			Providing this parameter with a *past* time will cause the status to be backdated,
+//			and will not push it to the user's followers. This is intended for importing old statuses.
 //		type: string
+//		format: date-time
 //		in: formData
 //	-
 //		name: language
@@ -380,12 +384,6 @@ func parseStatusCreateForm(c *gin.Context) (*apimodel.StatusCreateRequest, gtser
 		return nil, gtserror.NewErrorNotAcceptable(errors.New(text), text)
 	}
 
-	// Check not scheduled status.
-	if form.ScheduledAt != "" {
-		const text = "scheduled_at is not yet implemented"
-		return nil, gtserror.NewErrorNotImplemented(errors.New(text), text)
-	}
-
 	// Check if the deprecated "federated" field was
 	// set in lieu of "local_only", and use it if so.
 	if form.LocalOnly == nil && form.Federated != nil { // nolint:staticcheck

internal/api/model/status.go

@@ -17,7 +17,11 @@
 
 package model
 
-import "github.com/superseriousbusiness/gotosocial/internal/language"
+import (
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/language"
+)
 
 // Status models a status or post.
 //
@@ -231,9 +235,14 @@ type StatusCreateRequest struct {
 	Federated *bool `form:"federated" json:"federated"`
 
 	// ISO 8601 Datetime at which to schedule a status.
-	// Providing this parameter will cause ScheduledStatus to be returned instead of Status.
+	//
+	// Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status.
 	// Must be at least 5 minutes in the future.
-	ScheduledAt string `form:"scheduled_at" json:"scheduled_at"`
+	// This feature isn't implemented yet.
+	//
+	// Providing this parameter with a *past* time will cause the status to be backdated,
+	// and will not push it to the user's followers. This is intended for importing old statuses.
+	ScheduledAt *time.Time `form:"scheduled_at" json:"scheduled_at"`
 
 	// ISO 639 language code for this status.
 	Language string `form:"language" json:"language"`

internal/gtsmodel/status.go

@@ -86,7 +86,7 @@ func (s *Status) GetAccountID() string {
 	return s.AccountID
 }
 
-// GetBoostID implements timeline.Timelineable{}.
+// GetBoostOfID implements timeline.Timelineable{}.
 func (s *Status) GetBoostOfID() string {
 	return s.BoostOfID
 }
@@ -171,7 +171,7 @@ func (s *Status) EditsPopulated() bool {
 	return true
 }
 
-// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date
+// EmojisUpToDate returns whether status emoji attachments of receiving status are up-to-date
 // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't
 // use IDs as this is used to determine whether there are new emojis to fetch.
 func (s *Status) EmojisUpToDate(other *Status) bool {
@@ -386,3 +386,8 @@ type Content struct {
 	Content    string
 	ContentMap map[string]string
 }
+
+// BackfillStatus is a wrapper for creating a status without pushing notifications to followers.
+type BackfillStatus struct {
+	*Status
+}

internal/processing/status/create.go

@@ -19,10 +19,13 @@
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
@@ -92,11 +95,35 @@ func (p *Processor) Create(
 	// Get current time.
 	now := time.Now()
 
+	// Default to current time as creation time.
+	createdAt := now
+
+	// Handle backfilled/scheduled statuses.
+	backfill := false
+	if form.ScheduledAt != nil {
+		scheduledAt := *form.ScheduledAt
+
+		// Statuses may only be scheduled a minimum time into the future.
+		if now.Before(scheduledAt) {
+			const errText = "scheduled statuses are not yet supported"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorBadRequest(err, errText)
+		}
+
+		// If not scheduled into the future, this status is being backfilled.
+		backfill = true
+		createdAt = scheduledAt
+		var err error
+		if statusID, err = p.backfilledStatusID(ctx, createdAt); err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+	}
+
 	status := &gtsmodel.Status{
 		ID:                       statusID,
 		URI:                      accountURIs.StatusesURI + "/" + statusID,
 		URL:                      accountURIs.StatusesURL + "/" + statusID,
-		CreatedAt:                now,
+		CreatedAt:                createdAt,
 		Local:                    util.Ptr(true),
 		Account:                  requester,
 		AccountID:                requester.ID,
@@ -139,6 +166,7 @@ func (p *Processor) Create(
 		requester,
 		status,
 		form.InReplyToID,
+		backfill,
 	); errWithCode != nil {
 		return nil, errWithCode
 	}
@@ -165,11 +193,17 @@ func (p *Processor) Create(
 	}
 
 	if form.Poll != nil {
+		if backfill {
+			const errText = "posts with polls can't be backfilled"
+			err := gtserror.New(errText)
+			return nil, gtserror.NewErrorBadRequest(err, errText)
+		}
+
 		// Process poll, inserting into database.
 		poll, errWithCode := p.processPoll(ctx,
 			statusID,
 			form.Poll,
-			now,
+			createdAt,
 		)
 		if errWithCode != nil {
 			return nil, errWithCode
@@ -200,10 +234,14 @@ func (p *Processor) Create(
 	}
 
 	// Send it to the client API worker for async side-effects.
+	var model any = status
+	if backfill {
+		model = &gtsmodel.BackfillStatus{Status: status}
+	}
 	p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{
 		APObjectType:   ap.ObjectNote,
 		APActivityType: ap.ActivityCreate,
-		GTSModel:       status,
+		GTSModel:       model,
 		Origin:         requester,
 	})
 
@@ -227,7 +265,40 @@ func (p *Processor) Create(
 	return p.c.GetAPIStatus(ctx, requester, status)
 }
 
-func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status, inReplyToID string) gtserror.WithCode {
+// backfilledStatusID tries to find an unused ULID for a backfilled status.
+func (p *Processor) backfilledStatusID(ctx context.Context, createdAt time.Time) (string, error) {
+	// backfilledStatusIDRetries should be more than enough attempts.
+	const backfilledStatusIDRetries = 100
+
+	for try := 0; try < backfilledStatusIDRetries; try++ {
+		var err error
+
+		// Generate a ULID based on the backfilled status's original creation time.
+		statusID := id.NewULIDFromTime(createdAt)
+
+		// Check for an existing status with that ID.
+		_, err = p.state.DB.GetStatusByID(gtscontext.SetBarebones(ctx), statusID)
+		if errors.Is(err, db.ErrNoEntries) {
+			// We found an unused one.
+			return statusID, nil
+		} else if err != nil {
+			err := gtserror.Newf("DB error checking if a status ID was in use: %w", err)
+			return "", err
+		}
+		// That status ID is in use. Try again.
+	}
+
+	err := gtserror.Newf("failed to find an unused ID after %d tries", backfilledStatusIDRetries)
+	return "", err
+}
+
+func (p *Processor) processInReplyTo(
+	ctx context.Context,
+	requester *gtsmodel.Account,
+	status *gtsmodel.Status,
+	inReplyToID string,
+	backfill bool,
+) gtserror.WithCode {
 	if inReplyToID == "" {
 		// Not a reply.
 		// Nothing to do.
@@ -269,6 +340,13 @@ func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Ac
 		return gtserror.NewErrorForbidden(err, errText)
 	}
 
+	// When backfilling, only self-replies are allowed.
+	if backfill && requester.ID != inReplyTo.AccountID {
+		const errText = "replies to others can't be backfilled"
+		err := gtserror.New(errText)
+		return gtserror.NewErrorBadRequest(err, errText)
+	}
+
 	// Derive pendingApproval status.
 	var pendingApproval bool
 	switch {

internal/processing/workers/fromclientapi.go

@@ -260,9 +260,16 @@ func (p *clientAPI) CreateUser(ctx context.Context, cMsg *messages.FromClientAPI
 }
 
 func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientAPI) error {
-	status, ok := cMsg.GTSModel.(*gtsmodel.Status)
-	if !ok {
-		return gtserror.Newf("%T not parseable as *gtsmodel.Status", cMsg.GTSModel)
+	var status *gtsmodel.Status
+	backfill := false
+	if backfillStatus, ok := cMsg.GTSModel.(*gtsmodel.BackfillStatus); ok {
+		status = backfillStatus.Status
+		backfill = true
+	} else {
+		status, ok = cMsg.GTSModel.(*gtsmodel.Status)
+		if !ok {
+			return gtserror.Newf("%T not parseable as *gtsmodel.Status or *gtsmodel.BackfillStatus", cMsg.GTSModel)
+		}
 	}
 
 	// If pending approval is true then status must
@@ -344,12 +351,14 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA
 		log.Errorf(ctx, "error updating account stats: %v", err)
 	}
 
-	if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
-		log.Errorf(ctx, "error timelining and notifying status: %v", err)
-	}
+	if !backfill {
+		if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil {
+			log.Errorf(ctx, "error timelining and notifying status: %v", err)
+		}
 
-	if err := p.federate.CreateStatus(ctx, status); err != nil {
-		log.Errorf(ctx, "error federating status: %v", err)
+		if err := p.federate.CreateStatus(ctx, status); err != nil {
+			log.Errorf(ctx, "error federating status: %v", err)
+		}
 	}
 
 	if status.InReplyToID != "" {