[bugfix] visibility after implicit approval not getting invalidated (#3370)

* replicate issue

* update go-structr to v0.8.10 with internal linked-list fix, small tweaks to caching of interaction requests

* remove debug function

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
kim 2024-09-28 20:47:46 +00:00 committed by GitHub
parent 18b7e00fef
commit 095663f5cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 239 additions and 136 deletions

2
go.mod
View file

@ -22,7 +22,7 @@ require (
codeberg.org/gruf/go-runners v1.6.3 codeberg.org/gruf/go-runners v1.6.3
codeberg.org/gruf/go-sched v1.2.4 codeberg.org/gruf/go-sched v1.2.4
codeberg.org/gruf/go-storage v0.2.0 codeberg.org/gruf/go-storage v0.2.0
codeberg.org/gruf/go-structr v0.8.9 codeberg.org/gruf/go-structr v0.8.10
codeberg.org/superseriousbusiness/exif-terminator v0.9.0 codeberg.org/superseriousbusiness/exif-terminator v0.9.0
github.com/DmitriyVTitov/size v1.5.0 github.com/DmitriyVTitov/size v1.5.0
github.com/KimMachineGun/automemlimit v0.6.1 github.com/KimMachineGun/automemlimit v0.6.1

4
go.sum
View file

@ -72,8 +72,8 @@ codeberg.org/gruf/go-sched v1.2.4 h1:ddBB9o0D/2oU8NbQ0ldN5aWxogpXPRBATWi58+p++Hw
codeberg.org/gruf/go-sched v1.2.4/go.mod h1:wad6l+OcYGWMA2TzNLMmLObsrbBDxdJfEy5WvTgBjNk= codeberg.org/gruf/go-sched v1.2.4/go.mod h1:wad6l+OcYGWMA2TzNLMmLObsrbBDxdJfEy5WvTgBjNk=
codeberg.org/gruf/go-storage v0.2.0 h1:mKj3Lx6AavEkuXXtxqPhdq+akW9YwrnP16yQBF7K5ZI= codeberg.org/gruf/go-storage v0.2.0 h1:mKj3Lx6AavEkuXXtxqPhdq+akW9YwrnP16yQBF7K5ZI=
codeberg.org/gruf/go-storage v0.2.0/go.mod h1:o3GzMDE5QNUaRnm/daUzFqvuAaC4utlgXDXYO79sWKU= codeberg.org/gruf/go-storage v0.2.0/go.mod h1:o3GzMDE5QNUaRnm/daUzFqvuAaC4utlgXDXYO79sWKU=
codeberg.org/gruf/go-structr v0.8.9 h1:OyiSspWYCeJOm356fFPd+bDRumPrard2VAUXAPqZiJ0= codeberg.org/gruf/go-structr v0.8.10 h1:uSapW97/StRnYEhCtycaM0isCsEMYC+tx/knYr6SiVo=
codeberg.org/gruf/go-structr v0.8.9/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM= codeberg.org/gruf/go-structr v0.8.10/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM=
codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go= codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go=
codeberg.org/superseriousbusiness/exif-terminator v0.9.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE= codeberg.org/superseriousbusiness/exif-terminator v0.9.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

View file

@ -73,7 +73,7 @@ func (suite *AccountStatusesTestSuite) TestGetStatusesPublicOnly() {
suite.Equal(apimodel.VisibilityPublic, s.Visibility) suite.Equal(apimodel.VisibilityPublic, s.Visibility)
} }
suite.Equal(`<http://localhost:8080/api/v1/accounts/01F8MH17FWEB39HZJ76B6VXSKF/statuses?limit=20&max_id=01F8MH75CBF9JFX4ZAD54N0W0R&exclude_replies=false&exclude_reblogs=false&pinned=false&only_media=false&only_public=true>; rel="next", <http://localhost:8080/api/v1/accounts/01F8MH17FWEB39HZJ76B6VXSKF/statuses?limit=20&min_id=01G36SF3V6Y6V5BF9P4R7PQG7G&exclude_replies=false&exclude_reblogs=false&pinned=false&only_media=false&only_public=true>; rel="prev"`, result.Header.Get("link")) suite.Equal(`<http://localhost:8080/api/v1/accounts/01F8MH17FWEB39HZJ76B6VXSKF/statuses?limit=20&max_id=01F8MH75CBF9JFX4ZAD54N0W0R&exclude_replies=false&exclude_reblogs=false&pinned=false&only_media=false&only_public=true>; rel="next", <http://localhost:8080/api/v1/accounts/01F8MH17FWEB39HZJ76B6VXSKF/statuses?limit=20&min_id=01J5QVB9VC76NPPRQ207GG4DRZ&exclude_replies=false&exclude_reblogs=false&pinned=false&only_media=false&only_public=true>; rel="prev"`, result.Header.Get("link"))
} }
func (suite *AccountStatusesTestSuite) TestGetStatusesPublicOnlyMediaOnly() { func (suite *AccountStatusesTestSuite) TestGetStatusesPublicOnlyMediaOnly() {

View file

@ -591,7 +591,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() {
"text": "Hi @1happyturtle, can I reply?", "text": "Hi @1happyturtle, can I reply?",
"uri": "http://localhost:8080/some/determinate/url", "uri": "http://localhost:8080/some/determinate/url",
"url": "http://localhost:8080/some/determinate/url", "url": "http://localhost:8080/some/determinate/url",
"visibility": "unlisted" "visibility": "public"
}, },
"reblogged": true, "reblogged": true,
"reblogs_count": 0, "reblogs_count": 0,
@ -601,7 +601,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostImplicitAccept() {
"tags": [], "tags": [],
"uri": "http://localhost:8080/some/determinate/url", "uri": "http://localhost:8080/some/determinate/url",
"url": "http://localhost:8080/some/determinate/url", "url": "http://localhost:8080/some/determinate/url",
"visibility": "unlisted" "visibility": "public"
}`, out) }`, out)
// Target status should no // Target status should no

View file

@ -27,6 +27,7 @@
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
@ -185,13 +186,24 @@ func (suite *StatusFaveTestSuite) TestPostUnfaveable() {
// Fave a status that's pending approval by us. // Fave a status that's pending approval by us.
func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() { func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() {
var ( var (
ctx = context.Background()
targetStatus = suite.testStatuses["admin_account_status_5"] targetStatus = suite.testStatuses["admin_account_status_5"]
app = suite.testApplications["application_1"] app = suite.testApplications["application_1"]
token = suite.testTokens["local_account_2"] token = suite.testTokens["local_account_2"]
user = suite.testUsers["local_account_2"] user = suite.testUsers["local_account_2"]
account = suite.testAccounts["local_account_2"] account = suite.testAccounts["local_account_2"]
visFilter = visibility.NewFilter(&suite.state)
) )
// Check visibility of status to public before posting fave.
visible, err := visFilter.StatusVisible(ctx, nil, targetStatus)
if err != nil {
suite.FailNow(err.Error())
}
if visible {
suite.FailNow("status should not be visible yet")
}
out, recorder := suite.postStatusFave( out, recorder := suite.postStatusFave(
targetStatus.ID, targetStatus.ID,
app, app,
@ -268,30 +280,40 @@ func (suite *StatusFaveTestSuite) TestPostFaveImplicitAccept() {
"text": "Hi @1happyturtle, can I reply?", "text": "Hi @1happyturtle, can I reply?",
"uri": "http://localhost:8080/some/determinate/url", "uri": "http://localhost:8080/some/determinate/url",
"url": "http://localhost:8080/some/determinate/url", "url": "http://localhost:8080/some/determinate/url",
"visibility": "unlisted" "visibility": "public"
}`, out) }`, out)
// Target status should no // Target status should no
// longer be pending approval. // longer be pending approval.
dbStatus, err := suite.state.DB.GetStatusByID( dbStatus, err := suite.state.DB.GetStatusByID(
context.Background(), ctx,
targetStatus.ID, targetStatus.ID,
) )
if err != nil { if err != nil {
suite.FailNow(err.Error()) suite.FailNow(err.Error())
} }
suite.False(*dbStatus.PendingApproval) suite.False(*dbStatus.PendingApproval)
suite.NotEmpty(dbStatus.ApprovedByURI)
// There should be an Accept // There should be an Accept
// stored for the target status. // stored for the target status.
intReq, err := suite.state.DB.GetInteractionRequestByInteractionURI( intReq, err := suite.state.DB.GetInteractionRequestByInteractionURI(
context.Background(), targetStatus.URI, ctx, targetStatus.URI,
) )
if err != nil { if err != nil {
suite.FailNow(err.Error()) suite.FailNow(err.Error())
} }
suite.NotZero(intReq.AcceptedAt) suite.NotZero(intReq.AcceptedAt)
suite.NotEmpty(intReq.URI) suite.NotEmpty(intReq.URI)
// Check visibility of status to public after posting fave.
visible, err = visFilter.StatusVisible(ctx, nil, dbStatus)
if err != nil {
suite.FailNow(err.Error())
}
if !visible {
suite.FailNow("status should be visible")
}
} }
func TestStatusFaveTestSuite(t *testing.T) { func TestStatusFaveTestSuite(t *testing.T) {

View file

@ -18,7 +18,6 @@
package cache package cache
import ( import (
"database/sql"
"errors" "errors"
"time" "time"
@ -42,7 +41,6 @@ func ignoreErrors(err error) bool {
// (until invalidation). // (until invalidation).
db.ErrNoEntries, db.ErrNoEntries,
db.ErrAlreadyExists, db.ErrAlreadyExists,
sql.ErrNoRows,
) )
} }

View file

@ -48,9 +48,15 @@ func (c *Caches) initVisibility() {
{Fields: "RequesterID", Multiple: true}, {Fields: "RequesterID", Multiple: true},
{Fields: "Type,RequesterID,ItemID"}, {Fields: "Type,RequesterID,ItemID"},
}, },
MaxSize: cap, MaxSize: cap,
IgnoreErr: ignoreErrors, IgnoreErr: func(err error) bool {
Copy: copyF, // don't cache any errors,
// it gets a little too tricky
// otherwise with ensuring
// errors are cleared out
return true
},
Copy: copyF,
}) })
} }

View file

@ -26,8 +26,10 @@
"github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/util"
"github.com/uptrace/bun" "github.com/uptrace/bun"
) )
@ -84,6 +86,53 @@ func(request *gtsmodel.InteractionRequest) error {
) )
} }
func (i *interactionDB) GetInteractionRequestsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.InteractionRequest, error) {
// Load all interaction request IDs via cache loader callbacks.
requests, err := i.state.Caches.DB.InteractionRequest.LoadIDs("ID",
ids,
func(uncached []string) ([]*gtsmodel.InteractionRequest, error) {
// Preallocate expected length of uncached interaction requests.
requests := make([]*gtsmodel.InteractionRequest, 0, len(uncached))
// Perform database query scanning
// the remaining (uncached) IDs.
if err := i.db.NewSelect().
Model(&requests).
Where("? IN (?)", bun.Ident("id"), bun.In(uncached)).
Scan(ctx); err != nil {
return nil, err
}
return requests, nil
},
)
if err != nil {
return nil, err
}
// Reorder the requests by their
// IDs to ensure in correct order.
getID := func(r *gtsmodel.InteractionRequest) string { return r.ID }
util.OrderBy(requests, ids, getID)
if gtscontext.Barebones(ctx) {
// no need to fully populate.
return requests, nil
}
// Populate all loaded interaction requests, removing those we
// fail to populate (removes needing so many nil checks everywhere).
requests = slices.DeleteFunc(requests, func(request *gtsmodel.InteractionRequest) bool {
if err := i.PopulateInteractionRequest(ctx, request); err != nil {
log.Errorf(ctx, "error populating %s: %v", request.ID, err)
return true
}
return false
})
return requests, nil
}
func (i *interactionDB) getInteractionRequest( func (i *interactionDB) getInteractionRequest(
ctx context.Context, ctx context.Context,
lookup string, lookup string,
@ -205,13 +254,18 @@ func (i *interactionDB) UpdateInteractionRequest(ctx context.Context, request *g
} }
func (i *interactionDB) DeleteInteractionRequestByID(ctx context.Context, id string) error { func (i *interactionDB) DeleteInteractionRequestByID(ctx context.Context, id string) error {
defer i.state.Caches.DB.InteractionRequest.Invalidate("ID", id) // Delete interaction request by ID.
if _, err := i.db.NewDelete().
Table("interaction_requests").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return err
}
_, err := i.db.NewDelete(). // Invalidate cached interaction request with ID.
TableExpr("? AS ?", bun.Ident("interaction_requests"), bun.Ident("interaction_request")). i.state.Caches.DB.InteractionRequest.Invalidate("ID", id)
Where("? = ?", bun.Ident("interaction_request.id"), id).
Exec(ctx) return nil
return err
} }
func (i *interactionDB) GetInteractionsRequestsForAcct( func (i *interactionDB) GetInteractionsRequestsForAcct(
@ -317,19 +371,8 @@ func (i *interactionDB) GetInteractionsRequestsForAcct(
slices.Reverse(reqIDs) slices.Reverse(reqIDs)
} }
// For each interaction request ID, // Load all interaction requests by their IDs.
// select the interaction request. return i.GetInteractionRequestsByIDs(ctx, reqIDs)
reqs := make([]*gtsmodel.InteractionRequest, 0, len(reqIDs))
for _, id := range reqIDs {
req, err := i.GetInteractionRequestByID(ctx, id)
if err != nil {
return nil, err
}
reqs = append(reqs, req)
}
return reqs, nil
} }
func (i *interactionDB) IsInteractionRejected(ctx context.Context, interactionURI string) (bool, error) { func (i *interactionDB) IsInteractionRejected(ctx context.Context, interactionURI string) (bool, error) {

View file

@ -74,7 +74,8 @@ func (suite *TimelineTestSuite) publicCount() int {
var publicCount int var publicCount int
for _, status := range suite.testStatuses { for _, status := range suite.testStatuses {
if status.Visibility == gtsmodel.VisibilityPublic && if status.Visibility == gtsmodel.VisibilityPublic &&
status.BoostOfID == "" { status.BoostOfID == "" &&
!util.PtrOrZero(status.PendingApproval) {
publicCount++ publicCount++
} }
} }

View file

@ -168,6 +168,9 @@ func (suite *StatusVisibleTestSuite) TestVisiblePending() {
testStatus := new(gtsmodel.Status) testStatus := new(gtsmodel.Status)
*testStatus = *suite.testStatuses["admin_account_status_3"] *testStatus = *suite.testStatuses["admin_account_status_3"]
testStatus.PendingApproval = util.Ptr(true) testStatus.PendingApproval = util.Ptr(true)
if err := suite.state.DB.UpdateStatus(ctx, testStatus); err != nil {
suite.FailNow(err.Error())
}
for _, testCase := range []struct { for _, testCase := range []struct {
acct *gtsmodel.Account acct *gtsmodel.Account
@ -198,6 +201,43 @@ func (suite *StatusVisibleTestSuite) TestVisiblePending() {
suite.NoError(err) suite.NoError(err)
suite.Equal(testCase.visible, visible) suite.Equal(testCase.visible, visible)
} }
// Update the status to mark it as approved.
testStatus.PendingApproval = util.Ptr(false)
testStatus.ApprovedByURI = "http://localhost:8080/some/accept/uri"
if err := suite.state.DB.UpdateStatus(ctx, testStatus); err != nil {
suite.FailNow(err.Error())
}
for _, testCase := range []struct {
acct *gtsmodel.Account
visible bool
}{
{
acct: suite.testAccounts["admin_account"],
visible: true, // Own status, always visible.
},
{
acct: suite.testAccounts["local_account_1"],
visible: true, // Reply to zork, always visible.
},
{
acct: suite.testAccounts["local_account_2"],
visible: true, // Should be visible now.
},
{
acct: suite.testAccounts["remote_account_1"],
visible: true, // Should be visible now.
},
{
acct: nil, // Unauthed request.
visible: true, // Should be visible now (public status).
},
} {
visible, err := suite.filter.StatusVisible(ctx, testCase.acct, testStatus)
suite.NoError(err)
suite.Equal(testCase.visible, visible)
}
} }
func (suite *StatusVisibleTestSuite) TestVisibleLocalOnly() { func (suite *StatusVisibleTestSuite) TestVisibleLocalOnly() {

View file

@ -1744,7 +1744,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval()
"in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF",
"sensitive": false, "sensitive": false,
"spoiler_text": "", "spoiler_text": "",
"visibility": "unlisted", "visibility": "public",
"language": null, "language": null,
"uri": "http://localhost:8080/users/admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ", "uri": "http://localhost:8080/users/admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ",
"url": "http://localhost:8080/@admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ", "url": "http://localhost:8080/@admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ",
@ -3177,7 +3177,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() {
"in_reply_to_account_id": null, "in_reply_to_account_id": null,
"sensitive": true, "sensitive": true,
"spoiler_text": "you won't be able to reply to this without my approval", "spoiler_text": "you won't be able to reply to this without my approval",
"visibility": "unlisted", "visibility": "public",
"language": "en", "language": "en",
"uri": "http://localhost:8080/users/1happyturtle/statuses/01F8MHC8VWDRBQR0N1BATDDEM5", "uri": "http://localhost:8080/users/1happyturtle/statuses/01F8MHC8VWDRBQR0N1BATDDEM5",
"url": "http://localhost:8080/@1happyturtle/statuses/01F8MHC8VWDRBQR0N1BATDDEM5", "url": "http://localhost:8080/@1happyturtle/statuses/01F8MHC8VWDRBQR0N1BATDDEM5",
@ -3269,7 +3269,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() {
"in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF", "in_reply_to_account_id": "01F8MH5NBDF2MV7CTC4Q5128HF",
"sensitive": false, "sensitive": false,
"spoiler_text": "", "spoiler_text": "",
"visibility": "unlisted", "visibility": "public",
"language": null, "language": null,
"uri": "http://localhost:8080/users/admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ", "uri": "http://localhost:8080/users/admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ",
"url": "http://localhost:8080/@admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ", "url": "http://localhost:8080/@admin/statuses/01J5QVB9VC76NPPRQ207GG4DRZ",

View file

@ -1531,7 +1531,7 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
BoostOfID: "", BoostOfID: "",
BoostOfAccountID: "", BoostOfAccountID: "",
ThreadID: "01HCWE4P0EW9HBA5WHW97D5YV0", ThreadID: "01HCWE4P0EW9HBA5WHW97D5YV0",
Visibility: gtsmodel.VisibilityUnlocked, Visibility: gtsmodel.VisibilityPublic,
Sensitive: util.Ptr(false), Sensitive: util.Ptr(false),
CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
Federated: util.Ptr(true), Federated: util.Ptr(true),
@ -1811,7 +1811,7 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
BoostOfID: "", BoostOfID: "",
ThreadID: "01HCWE4P0EW9HBA5WHW97D5YV0", ThreadID: "01HCWE4P0EW9HBA5WHW97D5YV0",
ContentWarning: "you won't be able to reply to this without my approval", ContentWarning: "you won't be able to reply to this without my approval",
Visibility: gtsmodel.VisibilityUnlocked, Visibility: gtsmodel.VisibilityPublic,
Sensitive: util.Ptr(true), Sensitive: util.Ptr(true),
Language: "en", Language: "en",
CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",

View file

@ -575,8 +575,9 @@ func (c *Cache[T]) store_value(index *Index, key string, value T) {
item.data = value item.data = value
if index != nil { if index != nil {
// Append item to index. // Append item to index a key
index.append(key, item) // was already generated for.
index.append(&c.lru, key, item)
} }
// Get ptr to value data. // Get ptr to value data.
@ -607,8 +608,8 @@ func (c *Cache[T]) store_value(index *Index, key string, value T) {
continue continue
} }
// Append item to index. // Append item to this index.
idx.append(key, item) idx.append(&c.lru, key, item)
} }
// Add item to main lru list. // Add item to main lru list.
@ -645,8 +646,9 @@ func (c *Cache[T]) store_error(index *Index, key string, err error) {
// Set error val. // Set error val.
item.data = err item.data = err
// Append item to index. // Append item to index a key
index.append(key, item) // was already generated for.
index.append(&c.lru, key, item)
// Add item to main lru list. // Add item to main lru list.
c.lru.push_front(&item.elem) c.lru.push_front(&item.elem)

View file

@ -174,7 +174,7 @@ func (i *Index) init(t reflect.Type, cfg IndexConfig, cap int) {
// get_one will fetch one indexed item under key. // get_one will fetch one indexed item under key.
func (i *Index) get_one(key Key) *indexed_item { func (i *Index) get_one(key Key) *indexed_item {
// Get list at hash. // Get list at hash.
l, _ := i.data.Get(key.key) l := i.data.Get(key.key)
if l == nil { if l == nil {
return nil return nil
} }
@ -192,7 +192,7 @@ func (i *Index) get(key string, hook func(*indexed_item)) {
} }
// Get list at hash. // Get list at hash.
l, _ := i.data.Get(key) l := i.data.Get(key)
if l == nil { if l == nil {
return return
} }
@ -237,11 +237,12 @@ func (i *Index) key(buf *byteutil.Buffer, parts []unsafe.Pointer) string {
} }
// append will append the given index entry to appropriate // append will append the given index entry to appropriate
// doubly-linked-list in index hashmap. this handles case // doubly-linked-list in index hashmap. this handles case of
// of key collisions and overwriting 'unique' entries. // overwriting "unique" index entries, and removes from given
func (i *Index) append(key string, item *indexed_item) { // outer linked-list in the case that it is no longer indexed.
func (i *Index) append(ll *list, key string, item *indexed_item) {
// Look for existing. // Look for existing.
l, _ := i.data.Get(key) l := i.data.Get(key)
if l == nil { if l == nil {
@ -255,12 +256,21 @@ func (i *Index) append(key string, item *indexed_item) {
elem := l.head elem := l.head
l.remove(elem) l.remove(elem)
// Drop index from inner item. // Drop index from inner item,
// catching the evicted item.
e := (*index_entry)(elem.data) e := (*index_entry)(elem.data)
e.item.drop_index(e) evicted := e.item
evicted.drop_index(e)
// Free unused entry. // Free unused entry.
free_index_entry(e) free_index_entry(e)
if len(evicted.indexed) == 0 {
// Evicted item is not indexed,
// remove from outer linked list.
ll.remove(&evicted.elem)
free_indexed_item(evicted)
}
} }
// Prepare new index entry. // Prepare new index entry.
@ -283,7 +293,7 @@ func (i *Index) delete(key string, hook func(*indexed_item)) {
} }
// Get list at hash. // Get list at hash.
l, _ := i.data.Get(key) l := i.data.Get(key)
if l == nil { if l == nil {
return return
} }
@ -292,10 +302,9 @@ func (i *Index) delete(key string, hook func(*indexed_item)) {
i.data.Delete(key) i.data.Delete(key)
// Iterate entries in list. // Iterate entries in list.
for x := 0; x < l.len; x++ { l.rangefn(func(elem *list_elem) {
// Pop list head. // Remove elem.
elem := l.head
l.remove(elem) l.remove(elem)
// Extract element entry + item. // Extract element entry + item.
@ -310,7 +319,7 @@ func (i *Index) delete(key string, hook func(*indexed_item)) {
// Pass to hook. // Pass to hook.
hook(item) hook(item)
} })
// Release list. // Release list.
free_list(l) free_list(l)
@ -319,7 +328,7 @@ func (i *Index) delete(key string, hook func(*indexed_item)) {
// delete_entry deletes the given index entry. // delete_entry deletes the given index entry.
func (i *Index) delete_entry(entry *index_entry) { func (i *Index) delete_entry(entry *index_entry) {
// Get list at hash sum. // Get list at hash sum.
l, _ := i.data.Get(entry.key) l := i.data.Get(entry.key)
if l == nil { if l == nil {
return return
} }

View file

@ -50,12 +50,9 @@ func (i *indexed_item) drop_index(entry *index_entry) {
continue continue
} }
// Unset tptr value to
// ensure GC can take it.
i.indexed[x] = nil
// Move all index entries down + reslice. // Move all index entries down + reslice.
_ = copy(i.indexed[x:], i.indexed[x+1:]) _ = copy(i.indexed[x:], i.indexed[x+1:])
i.indexed[len(i.indexed)-1] = nil
i.indexed = i.indexed[:len(i.indexed)-1] i.indexed = i.indexed[:len(i.indexed)-1]
break break
} }

View file

@ -48,27 +48,17 @@ func free_list(list *list) {
// push_front will push the given elem to front (head) of list. // push_front will push the given elem to front (head) of list.
func (l *list) push_front(elem *list_elem) { func (l *list) push_front(elem *list_elem) {
if l.len == 0 { // Set new head.
// Set new tail + head oldHead := l.head
l.head = elem l.head = elem
l.tail = elem
// Link elem to itself
elem.next = elem
elem.prev = elem
} else {
oldHead := l.head
if oldHead != nil {
// Link to old head // Link to old head
elem.next = oldHead elem.next = oldHead
oldHead.prev = elem oldHead.prev = elem
} else {
// Link up to tail // First in list.
elem.prev = l.tail l.tail = elem
l.tail.next = elem
// Set new head
l.head = elem
} }
// Incr count // Incr count
@ -77,27 +67,17 @@ func (l *list) push_front(elem *list_elem) {
// push_back will push the given elem to back (tail) of list. // push_back will push the given elem to back (tail) of list.
func (l *list) push_back(elem *list_elem) { func (l *list) push_back(elem *list_elem) {
if l.len == 0 { // Set new tail.
// Set new tail + head oldTail := l.tail
l.head = elem l.tail = elem
l.tail = elem
// Link elem to itself
elem.next = elem
elem.prev = elem
} else {
oldTail := l.tail
if oldTail != nil {
// Link to old tail // Link to old tail
elem.prev = oldTail elem.prev = oldTail
oldTail.next = elem oldTail.next = elem
} else {
// Link up to head // First in list.
elem.next = l.head l.head = elem
l.head.prev = elem
// Set new tail
l.tail = elem
} }
// Incr count // Incr count
@ -105,53 +85,57 @@ func (l *list) push_back(elem *list_elem) {
} }
// move_front will move given elem to front (head) of list. // move_front will move given elem to front (head) of list.
// if it is already at front this call is a no-op.
func (l *list) move_front(elem *list_elem) { func (l *list) move_front(elem *list_elem) {
if elem == l.head {
return
}
l.remove(elem) l.remove(elem)
l.push_front(elem) l.push_front(elem)
} }
// move_back will move given elem to back (tail) of list. // move_back will move given elem to back (tail) of list,
// if it is already at back this call is a no-op.
func (l *list) move_back(elem *list_elem) { func (l *list) move_back(elem *list_elem) {
if elem == l.tail {
return
}
l.remove(elem) l.remove(elem)
l.push_back(elem) l.push_back(elem)
} }
// remove will remove given elem from list. // remove will remove given elem from list.
func (l *list) remove(elem *list_elem) { func (l *list) remove(elem *list_elem) {
if l.len <= 1 { // Get linked elems.
// Drop elem's links
elem.next = nil
elem.prev = nil
// Only elem in list
l.head = nil
l.tail = nil
l.len = 0
return
}
// Get surrounding elems
next := elem.next next := elem.next
prev := elem.prev prev := elem.prev
// Relink chain // Unset elem.
next.prev = prev
prev.next = next
switch elem {
// Set new head
case l.head:
l.head = next
// Set new tail
case l.tail:
l.tail = prev
}
// Drop elem's links
elem.next = nil elem.next = nil
elem.prev = nil elem.prev = nil
switch {
// elem is ONLY one in list.
case next == nil && prev == nil:
l.head = nil
l.tail = nil
// elem is front in list.
case next != nil && prev == nil:
l.head = next
next.prev = nil
// elem is last in list.
case prev != nil && next == nil:
l.tail = prev
prev.next = nil
// elem in middle of list.
default:
next.prev = prev
prev.next = next
}
// Decr count // Decr count
l.len-- l.len--
} }
@ -161,9 +145,11 @@ func (l *list) rangefn(fn func(*list_elem)) {
if fn == nil { if fn == nil {
panic("nil fn") panic("nil fn")
} }
elem := l.head for e := l.head; //
for i := 0; i < l.len; i++ { e != nil; //
fn(elem) {
elem = elem.next n := e.next
fn(e)
e = n
} }
} }

View file

@ -10,9 +10,8 @@ func (m *hashmap) init(cap int) {
m.n = cap m.n = cap
} }
func (m *hashmap) Get(key string) (*list, bool) { func (m *hashmap) Get(key string) *list {
list, ok := m.m[key] return m.m[key]
return list, ok
} }
func (m *hashmap) Put(key string, list *list) { func (m *hashmap) Put(key string, list *list) {

View file

@ -308,8 +308,8 @@ func (q *Queue[T]) index(value T) *indexed_item {
continue continue
} }
// Append item to index. // Append item to this index.
idx.append(key, item) idx.append(&q.queue, key, item)
} }
// Done with buf. // Done with buf.

2
vendor/modules.txt vendored
View file

@ -66,7 +66,7 @@ codeberg.org/gruf/go-storage/disk
codeberg.org/gruf/go-storage/internal codeberg.org/gruf/go-storage/internal
codeberg.org/gruf/go-storage/memory codeberg.org/gruf/go-storage/memory
codeberg.org/gruf/go-storage/s3 codeberg.org/gruf/go-storage/s3
# codeberg.org/gruf/go-structr v0.8.9 # codeberg.org/gruf/go-structr v0.8.10
## explicit; go 1.21 ## explicit; go 1.21
codeberg.org/gruf/go-structr codeberg.org/gruf/go-structr
# codeberg.org/superseriousbusiness/exif-terminator v0.9.0 # codeberg.org/superseriousbusiness/exif-terminator v0.9.0