From 5e060d0bcb9af77e7b6d11acba99db6219d4f68e Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Sat, 10 Dec 2022 22:43:11 +0100 Subject: [PATCH] [feature] Start implementing refetch of lost media files via `/api/v1/admin/media_refetch` (#1221) * [chore] Move ShortcodeDomain to its own little util func * [feature] Add RefetchEmojis function to media manager * [feature] Expose admin media refresh via admin API * update following review feedback - change/fix log levels - make sure not to try to refetch local emojis - small style refactoring + comments * log on emoji refetch start Signed-off-by: kim Co-authored-by: kim --- docs/api/swagger.yaml | 34 +++++ internal/api/client/admin/admin.go | 4 + internal/api/client/admin/mediarefetch.go | 93 ++++++++++++++ internal/media/manager.go | 29 ++++- internal/media/media_test.go | 17 ++- internal/media/refetch.go | 149 ++++++++++++++++++++++ internal/media/refetch_test.go | 85 ++++++++++++ internal/processing/admin.go | 4 + internal/processing/admin/admin.go | 26 ++-- internal/processing/admin/getemojis.go | 8 +- internal/processing/admin/mediarefetch.go | 48 +++++++ internal/processing/processor.go | 4 +- internal/util/emoji.go | 26 ++++ 13 files changed, 499 insertions(+), 28 deletions(-) create mode 100644 internal/api/client/admin/mediarefetch.go create mode 100644 internal/media/refetch.go create mode 100644 internal/media/refetch_test.go create mode 100644 internal/processing/admin/mediarefetch.go create mode 100644 internal/util/emoji.go diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index ac2b1f303..fe0890fb5 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -3209,6 +3209,40 @@ paths: summary: Clean up remote media older than the specified number of days. tags: - admin + /api/v1/admin/media_refetch: + post: + description: |- + Currently, this only includes remote emojis. + This endpoint is useful when data loss has occurred, and you want to try to recover to a working state. + operationId: mediaRefetch + parameters: + - description: Domain to refetch media from. If empty, all domains will be refetched. + in: query + name: domain + type: string + produces: + - application/json + responses: + "202": + description: Request accepted and will be processed. Check the logs for progress / errors. + "400": + description: bad request + "401": + description: unauthorized + "403": + description: forbidden + "404": + description: not found + "406": + description: not acceptable + "500": + description: internal server error + security: + - OAuth2 Bearer: + - admin + summary: Refetch media specified in the database but missing from storage. + tags: + - admin /api/v1/apps: post: consumes: diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index 7032a5b95..569354c96 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -46,6 +46,7 @@ // AccountsActionPath is used for taking action on a single account. AccountsActionPath = AccountsPathWithID + "/action" MediaCleanupPath = BasePath + "/media_cleanup" + MediaRefetchPath = BasePath + "/media_refetch" // ExportQueryKey is for requesting a public export of some data. ExportQueryKey = "export" @@ -63,6 +64,8 @@ MinShortcodeDomainKey = "min_shortcode_domain" // LimitKey is for specifying maximum number of results to return. LimitKey = "limit" + // DomainQueryKey is for specifying a domain during admin actions. + DomainQueryKey = "domain" ) // Module implements the ClientAPIModule interface for admin-related actions (reports, emojis, etc) @@ -90,6 +93,7 @@ func (m *Module) Route(r router.Router) error { r.AttachHandler(http.MethodDelete, DomainBlocksPathWithID, m.DomainBlockDELETEHandler) r.AttachHandler(http.MethodPost, AccountsActionPath, m.AccountActionPOSTHandler) r.AttachHandler(http.MethodPost, MediaCleanupPath, m.MediaCleanupPOSTHandler) + r.AttachHandler(http.MethodPost, MediaRefetchPath, m.MediaRefetchPOSTHandler) r.AttachHandler(http.MethodGet, EmojiCategoriesPath, m.EmojiCategoriesGETHandler) return nil } diff --git a/internal/api/client/admin/mediarefetch.go b/internal/api/client/admin/mediarefetch.go new file mode 100644 index 000000000..9c8a30c1b --- /dev/null +++ b/internal/api/client/admin/mediarefetch.go @@ -0,0 +1,93 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/superseriousbusiness/gotosocial/internal/api" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// MediaRefetchPOSTHandler swagger:operation POST /api/v1/admin/media_refetch mediaRefetch +// +// Refetch media specified in the database but missing from storage. +// Currently, this only includes remote emojis. +// This endpoint is useful when data loss has occurred, and you want to try to recover to a working state. +// +// --- +// tags: +// - admin +// +// produces: +// - application/json +// +// security: +// - OAuth2 Bearer: +// - admin +// +// parameters: +// - +// name: domain +// in: query +// description: >- +// Domain to refetch media from. +// If empty, all domains will be refetched. +// type: string +// +// responses: +// '202': +// description: >- +// Request accepted and will be processed. +// Check the logs for progress / errors. +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '404': +// description: not found +// '406': +// description: not acceptable +// '500': +// description: internal server error +func (m *Module) MediaRefetchPOSTHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + api.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGet) + return + } + + if !*authed.User.Admin { + err := fmt.Errorf("user %s not an admin", authed.User.ID) + api.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGet) + return + } + + if errWithCode := m.processor.AdminMediaRefetch(c.Request.Context(), authed, c.Query(DomainQueryKey)); errWithCode != nil { + api.ErrorHandler(c, errWithCode, m.processor.InstanceGet) + return + } + + c.Status(http.StatusAccepted) +} diff --git a/internal/media/manager.go b/internal/media/manager.go index 67c03fb31..a9d71e589 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -40,6 +40,15 @@ // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { + // Stop stops the underlying worker pool of the manager. It should be called + // when closing GoToSocial in order to cleanly finish any in-progress jobs. + // It will block until workers are finished processing. + Stop() error + + /* + PROCESSING FUNCTIONS + */ + // ProcessMedia begins the process of decoding and storing the given data as an attachment. // It will return a pointer to a ProcessingMedia struct upon which further actions can be performed, such as getting // the finished media, thumbnail, attachment, etc. @@ -75,6 +84,10 @@ type Manager interface { // RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote. RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) + /* + PRUNING FUNCTIONS + */ + // PruneAllRemote prunes all remote media attachments cached on this instance which are older than the given amount of days. // 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size), // and setting 'cached' to false on the associated attachment. @@ -98,10 +111,18 @@ type Manager interface { // is returned to the caller. PruneOrphaned(ctx context.Context, dry bool) (int, error) - // Stop stops the underlying worker pool of the manager. It should be called - // when closing GoToSocial in order to cleanly finish any in-progress jobs. - // It will block until workers are finished processing. - Stop() error + /* + REFETCHING FUNCTIONS + Useful when data loss has occurred. + */ + + // RefetchEmojis iterates through remote emojis (for the given domain, or all if domain is empty string). + // + // For each emoji, the manager will check whether both the full size and static images are present in storage. + // If not, the manager will refetch and reprocess full size and static images for the emoji. + // + // The provided DereferenceMedia function will be used when it's necessary to refetch something this way. + RefetchEmojis(ctx context.Context, domain string, dereferenceMedia DereferenceMedia) (int, error) } type manager struct { diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 435994073..05408497c 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -20,22 +20,26 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/testrig" ) type MediaStandardTestSuite struct { suite.Suite - db db.DB - storage *storage.Driver - manager media.Manager - testAttachments map[string]*gtsmodel.MediaAttachment - testAccounts map[string]*gtsmodel.Account - testEmojis map[string]*gtsmodel.Emoji + db db.DB + storage *storage.Driver + manager media.Manager + transportController transport.Controller + testAttachments map[string]*gtsmodel.MediaAttachment + testAccounts map[string]*gtsmodel.Account + testEmojis map[string]*gtsmodel.Emoji } func (suite *MediaStandardTestSuite) SetupSuite() { @@ -53,6 +57,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.testAccounts = testrig.NewTestAccounts() suite.testEmojis = testrig.NewTestEmojis() suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage) + suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](0, 0)) } func (suite *MediaStandardTestSuite) TearDownTest() { diff --git a/internal/media/refetch.go b/internal/media/refetch.go new file mode 100644 index 000000000..abdec232a --- /dev/null +++ b/internal/media/refetch.go @@ -0,0 +1,149 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type DereferenceMedia func(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error) + +func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceMedia DereferenceMedia) (int, error) { + // normalize domain + if domain == "" { + domain = db.EmojiAllDomains + } + + var ( + maxShortcodeDomain string + refetchIDs []string + ) + + // page through emojis 20 at a time, looking for those with missing images + for { + // Fetch next block of emojis from database + emojis, err := m.db.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) + if err != nil { + if !errors.Is(err, db.ErrNoEntries) { + // an actual error has occurred + log.Errorf("error fetching emojis from database: %s", err) + } + break + } + + for _, emoji := range emojis { + if emoji.Domain == "" { + // never try to refetch local emojis + continue + } + + if refetch, err := m.emojiRequiresRefetch(ctx, emoji); err != nil { + // an error here indicates something is wrong with storage, so we should stop + return 0, fmt.Errorf("error checking refetch requirement for emoji %s: %w", util.ShortcodeDomain(emoji), err) + } else if !refetch { + continue + } + + refetchIDs = append(refetchIDs, emoji.ID) + } + + // Update next maxShortcodeDomain from last emoji + maxShortcodeDomain = util.ShortcodeDomain(emojis[len(emojis)-1]) + } + + // bail early if we've got nothing to do + toRefetchCount := len(refetchIDs) + if toRefetchCount == 0 { + log.Debug("no remote emojis require a refetch") + return 0, nil + } + log.Debugf("%d remote emoji(s) require a refetch, doing that now...", toRefetchCount) + + var totalRefetched int + for _, emojiID := range refetchIDs { + emoji, err := m.db.GetEmojiByID(ctx, emojiID) + if err != nil { + // this shouldn't happen--since we know we have the emoji--so return if it does + return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err) + } + shortcodeDomain := util.ShortcodeDomain(emoji) + + if emoji.ImageRemoteURL == "" { + log.Errorf("remote emoji %s could not be refreshed because it has no ImageRemoteURL set", shortcodeDomain) + continue + } + + emojiImageIRI, err := url.Parse(emoji.ImageRemoteURL) + if err != nil { + log.Errorf("remote emoji %s could not be refreshed because its ImageRemoteURL (%s) is not a valid uri: %s", shortcodeDomain, emoji.ImageRemoteURL, err) + continue + } + + dataFunc := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) { + return dereferenceMedia(ctx, emojiImageIRI) + } + + processingEmoji, err := m.ProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{ + Domain: &emoji.Domain, + ImageRemoteURL: &emoji.ImageRemoteURL, + ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL, + Disabled: emoji.Disabled, + VisibleInPicker: emoji.VisibleInPicker, + }, true) + if err != nil { + log.Errorf("emoji %s could not be refreshed because of an error during processing: %s", shortcodeDomain, err) + continue + } + + if _, err := processingEmoji.LoadEmoji(ctx); err != nil { + log.Errorf("emoji %s could not be refreshed because of an error during loading: %s", shortcodeDomain, err) + continue + } + + log.Tracef("refetched emoji %s successfully from remote", shortcodeDomain) + totalRefetched++ + } + + return totalRefetched, nil +} + +func (m *manager) emojiRequiresRefetch(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { + if has, err := m.storage.Has(ctx, emoji.ImagePath); err != nil { + return false, err + } else if !has { + return true, nil + } + + if has, err := m.storage.Has(ctx, emoji.ImageStaticPath); err != nil { + return false, err + } else if !has { + return true, nil + } + + return false, nil +} diff --git a/internal/media/refetch_test.go b/internal/media/refetch_test.go new file mode 100644 index 000000000..b6f1f3efc --- /dev/null +++ b/internal/media/refetch_test.go @@ -0,0 +1,85 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package media_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" +) + +type RefetchTestSuite struct { + MediaStandardTestSuite +} + +func (suite *RefetchTestSuite) TestRefetchEmojisNothingToDo() { + ctx := context.Background() + + adminAccount := suite.testAccounts["admin_account"] + transport, err := suite.transportController.NewTransportForUsername(ctx, adminAccount.Username) + if err != nil { + suite.FailNow(err.Error()) + } + + refetched, err := suite.manager.RefetchEmojis(ctx, "", transport.DereferenceMedia) + suite.NoError(err) + suite.Equal(0, refetched) +} + +func (suite *RefetchTestSuite) TestRefetchEmojis() { + ctx := context.Background() + + if err := suite.storage.Delete(ctx, suite.testEmojis["yell"].ImagePath); err != nil { + suite.FailNow(err.Error()) + } + + adminAccount := suite.testAccounts["admin_account"] + transport, err := suite.transportController.NewTransportForUsername(ctx, adminAccount.Username) + if err != nil { + suite.FailNow(err.Error()) + } + + refetched, err := suite.manager.RefetchEmojis(ctx, "", transport.DereferenceMedia) + suite.NoError(err) + suite.Equal(1, refetched) +} + +func (suite *RefetchTestSuite) TestRefetchEmojisLocal() { + ctx := context.Background() + + // delete the image for a LOCAL emoji + if err := suite.storage.Delete(ctx, suite.testEmojis["rainbow"].ImagePath); err != nil { + suite.FailNow(err.Error()) + } + + adminAccount := suite.testAccounts["admin_account"] + transport, err := suite.transportController.NewTransportForUsername(ctx, adminAccount.Username) + if err != nil { + suite.FailNow(err.Error()) + } + + refetched, err := suite.manager.RefetchEmojis(ctx, "", transport.DereferenceMedia) + suite.NoError(err) + suite.Equal(0, refetched) // shouldn't refetch anything because local +} + +func TestRefetchTestSuite(t *testing.T) { + suite.Run(t, &RefetchTestSuite{}) +} diff --git a/internal/processing/admin.go b/internal/processing/admin.go index 7b0933c0e..6654c9770 100644 --- a/internal/processing/admin.go +++ b/internal/processing/admin.go @@ -77,3 +77,7 @@ func (p *processor) AdminDomainBlockDelete(ctx context.Context, authed *oauth.Au func (p *processor) AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode { return p.adminProcessor.MediaPrune(ctx, mediaRemoteCacheDays) } + +func (p *processor) AdminMediaRefetch(ctx context.Context, authed *oauth.Auth, domain string) gtserror.WithCode { + return p.adminProcessor.MediaRefetch(ctx, authed.Account, domain) +} diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index f04d322ad..b0d4ce460 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -30,6 +30,7 @@ "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -48,23 +49,26 @@ type Processor interface { EmojiUpdate(ctx context.Context, id string, form *apimodel.EmojiUpdateRequest) (*apimodel.AdminEmoji, gtserror.WithCode) EmojiCategoriesGet(ctx context.Context) ([]*apimodel.EmojiCategory, gtserror.WithCode) MediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode + MediaRefetch(ctx context.Context, requestingAccount *gtsmodel.Account, domain string) gtserror.WithCode } type processor struct { - tc typeutils.TypeConverter - mediaManager media.Manager - storage *storage.Driver - clientWorker *concurrency.WorkerPool[messages.FromClientAPI] - db db.DB + tc typeutils.TypeConverter + mediaManager media.Manager + transportController transport.Controller + storage *storage.Driver + clientWorker *concurrency.WorkerPool[messages.FromClientAPI] + db db.DB } // New returns a new admin processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *storage.Driver, clientWorker *concurrency.WorkerPool[messages.FromClientAPI]) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *storage.Driver, clientWorker *concurrency.WorkerPool[messages.FromClientAPI]) Processor { return &processor{ - tc: tc, - mediaManager: mediaManager, - storage: storage, - clientWorker: clientWorker, - db: db, + tc: tc, + mediaManager: mediaManager, + transportController: transportController, + storage: storage, + clientWorker: clientWorker, + db: db, } } diff --git a/internal/processing/admin/getemojis.go b/internal/processing/admin/getemojis.go index d44b4d250..a15ce6e75 100644 --- a/internal/processing/admin/getemojis.go +++ b/internal/processing/admin/getemojis.go @@ -88,14 +88,10 @@ func (p *processor) EmojisGet(ctx context.Context, account *gtsmodel.Account, us Items: items, Path: "api/v1/admin/custom_emojis", NextMaxIDKey: "max_shortcode_domain", - NextMaxIDValue: shortcodeDomain(emojis[count-1]), + NextMaxIDValue: util.ShortcodeDomain(emojis[count-1]), PrevMinIDKey: "min_shortcode_domain", - PrevMinIDValue: shortcodeDomain(emojis[0]), + PrevMinIDValue: util.ShortcodeDomain(emojis[0]), Limit: limit, ExtraQueryParams: []string{filterBuilder.String()}, }) } - -func shortcodeDomain(emoji *gtsmodel.Emoji) string { - return emoji.Shortcode + "@" + emoji.Domain -} diff --git a/internal/processing/admin/mediarefetch.go b/internal/processing/admin/mediarefetch.go new file mode 100644 index 000000000..0870b645d --- /dev/null +++ b/internal/processing/admin/mediarefetch.go @@ -0,0 +1,48 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin + +import ( + "context" + "fmt" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" +) + +func (p *processor) MediaRefetch(ctx context.Context, requestingAccount *gtsmodel.Account, domain string) gtserror.WithCode { + transport, err := p.transportController.NewTransportForUsername(ctx, requestingAccount.Username) + if err != nil { + err = fmt.Errorf("error getting transport for user %s during media refetch request: %w", requestingAccount.Username, err) + return gtserror.NewErrorInternalError(err) + } + + go func() { + log.Info("starting emoji refetch") + refetched, err := p.mediaManager.RefetchEmojis(context.Background(), domain, transport.DereferenceMedia) + if err != nil { + log.Errorf("error refetching emojis: %s", err) + } else { + log.Infof("refetched %d emojis from remote", refetched) + } + }() + + return nil +} diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 88b0f5594..3067c56b7 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -136,6 +136,8 @@ type Processor interface { AdminDomainBlockDelete(ctx context.Context, authed *oauth.Auth, id string) (*apimodel.DomainBlock, gtserror.WithCode) // AdminMediaRemotePrune triggers a prune of remote media according to the given number of mediaRemoteCacheDays AdminMediaPrune(ctx context.Context, mediaRemoteCacheDays int) gtserror.WithCode + // AdminMediaRefetch triggers a refetch of remote media for the given domain (or all if domain is empty). + AdminMediaRefetch(ctx context.Context, authed *oauth.Auth, domain string) gtserror.WithCode // AppCreate processes the creation of a new API application AppCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.ApplicationCreateRequest) (*apimodel.Application, gtserror.WithCode) @@ -318,7 +320,7 @@ func NewProcessor( statusProcessor := status.New(db, tc, clientWorker, parseMentionFunc) streamingProcessor := streaming.New(db, oauthServer) accountProcessor := account.New(db, tc, mediaManager, oauthServer, clientWorker, federator, parseMentionFunc) - adminProcessor := admin.New(db, tc, mediaManager, storage, clientWorker) + adminProcessor := admin.New(db, tc, mediaManager, federator.TransportController(), storage, clientWorker) mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage) userProcessor := user.New(db, emailSender) federationProcessor := federationProcessor.New(db, tc, federator) diff --git a/internal/util/emoji.go b/internal/util/emoji.go new file mode 100644 index 000000000..4a0135cec --- /dev/null +++ b/internal/util/emoji.go @@ -0,0 +1,26 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package util + +import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + +// ShortcodeDomain returns the [shortcode]@[domain] for the given emoji. +func ShortcodeDomain(emoji *gtsmodel.Emoji) string { + return emoji.Shortcode + "@" + emoji.Domain +}