From 70739d32cc3894ce55f889c2e2fdb09d41e33df9 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Fri, 10 Feb 2023 20:16:01 +0000 Subject: [PATCH] [performance] remove throttling timers (#1466) * remove throttling timers, support setting retry-after, use retry-after in transport * remove unused variables * add throttling-retry-after to cmd flags * update envparsing to include new throttling-retry-after * update example config to include retry-after documentation * also support retry-after formatted as date-time, ensure max backoff time --------- Signed-off-by: kim --- cmd/gotosocial/action/server/server.go | 9 ++-- example/config.yaml | 7 +++ internal/config/config.go | 7 +-- internal/config/flags.go | 1 + internal/config/helpers.gen.go | 25 +++++++++ internal/middleware/throttling.go | 38 ++++---------- internal/transport/transport.go | 70 ++++++++++++++++---------- test/envparsing.sh | 3 +- 8 files changed, 98 insertions(+), 62 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 8cc31430b..7d4a1281d 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -203,10 +203,11 @@ // throttling cpuMultiplier := config.GetAdvancedThrottlingMultiplier() - clThrottle := middleware.Throttle(cpuMultiplier) // client api - s2sThrottle := middleware.Throttle(cpuMultiplier) // server-to-server (AP) - fsThrottle := middleware.Throttle(cpuMultiplier) // fileserver / web templates - pkThrottle := middleware.Throttle(cpuMultiplier) // throttle public key endpoint separately + retryAfter := config.GetAdvancedThrottlingRetryAfter() + clThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // client api + s2sThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // server-to-server (AP) + fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates + pkThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // throttle public key endpoint separately gzip := middleware.Gzip() // applied to all except fileserver diff --git a/example/config.yaml b/example/config.yaml index b411da4db..3b84feb70 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -760,3 +760,10 @@ advanced-rate-limit-requests: 300 # Examples: [8, 4, 9, 0] # Default: 8 advanced-throttling-multiplier: 8 + +# Duration. Time period to use as the "retry-after" header value in response to throttled requests. +# Minimum resolution is 1 second. +# +# Examples: [30s, 10s, 5s, 1m] +# Default: 30s +advanced-throttling-retry-after: "30s" \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 7025ab63a..516fb11d6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -133,9 +133,10 @@ type Configuration struct { SyslogProtocol string `name:"syslog-protocol" usage:"Protocol to use when directing logs to syslog. Leave empty to connect to local syslog."` SyslogAddress string `name:"syslog-address" usage:"Address:port to send syslog logs to. Leave empty to connect to local syslog."` - AdvancedCookiesSamesite string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"` - AdvancedRateLimitRequests int `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."` - AdvancedThrottlingMultiplier int `name:"advanced-throttling-multiplier" usage:"Multiplier to use per cpu for http request throttling. 0 or less turns throttling off."` + AdvancedCookiesSamesite string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"` + AdvancedRateLimitRequests int `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."` + AdvancedThrottlingMultiplier int `name:"advanced-throttling-multiplier" usage:"Multiplier to use per cpu for http request throttling. 0 or less turns throttling off."` + AdvancedThrottlingRetryAfter time.Duration `name:"advanced-throttling-retry-after" usage:"Retry-After duration response to send for throttled requests."` // Cache configuration vars. Cache CacheConfiguration `name:"cache"` diff --git a/internal/config/flags.go b/internal/config/flags.go index e594615d1..d8c31368b 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -139,6 +139,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) { cmd.Flags().String(AdvancedCookiesSamesiteFlag(), cfg.AdvancedCookiesSamesite, fieldtag("AdvancedCookiesSamesite", "usage")) cmd.Flags().Int(AdvancedRateLimitRequestsFlag(), cfg.AdvancedRateLimitRequests, fieldtag("AdvancedRateLimitRequests", "usage")) cmd.Flags().Int(AdvancedThrottlingMultiplierFlag(), cfg.AdvancedThrottlingMultiplier, fieldtag("AdvancedThrottlingMultiplier", "usage")) + cmd.Flags().Duration(AdvancedThrottlingRetryAfterFlag(), cfg.AdvancedThrottlingRetryAfter, fieldtag("AdvancedThrottlingRetryAfter", "usage")) }) } diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 3bf62efc6..64e7d02f7 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1999,6 +1999,31 @@ func GetAdvancedThrottlingMultiplier() int { return global.GetAdvancedThrottling // SetAdvancedThrottlingMultiplier safely sets the value for global configuration 'AdvancedThrottlingMultiplier' field func SetAdvancedThrottlingMultiplier(v int) { global.SetAdvancedThrottlingMultiplier(v) } +// GetAdvancedThrottlingRetryAfter safely fetches the Configuration value for state's 'AdvancedThrottlingRetryAfter' field +func (st *ConfigState) GetAdvancedThrottlingRetryAfter() (v time.Duration) { + st.mutex.Lock() + v = st.config.AdvancedThrottlingRetryAfter + st.mutex.Unlock() + return +} + +// SetAdvancedThrottlingRetryAfter safely sets the Configuration value for state's 'AdvancedThrottlingRetryAfter' field +func (st *ConfigState) SetAdvancedThrottlingRetryAfter(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.AdvancedThrottlingRetryAfter = v + st.reloadToViper() +} + +// AdvancedThrottlingRetryAfterFlag returns the flag name for the 'AdvancedThrottlingRetryAfter' field +func AdvancedThrottlingRetryAfterFlag() string { return "advanced-throttling-retry-after" } + +// GetAdvancedThrottlingRetryAfter safely fetches the value for global configuration 'AdvancedThrottlingRetryAfter' field +func GetAdvancedThrottlingRetryAfter() time.Duration { return global.GetAdvancedThrottlingRetryAfter() } + +// SetAdvancedThrottlingRetryAfter safely sets the value for global configuration 'AdvancedThrottlingRetryAfter' field +func SetAdvancedThrottlingRetryAfter(v time.Duration) { global.SetAdvancedThrottlingRetryAfter(v) } + // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) { st.mutex.Lock() diff --git a/internal/middleware/throttling.go b/internal/middleware/throttling.go index e7ad321f4..cc837f25e 100644 --- a/internal/middleware/throttling.go +++ b/internal/middleware/throttling.go @@ -29,17 +29,12 @@ import ( "net/http" "runtime" + "strconv" "time" "github.com/gin-gonic/gin" ) -const ( - errCapacityExceeded = "server capacity exceeded" - errTimedOut = "timed out while waiting for a pending request to complete" - errContextCanceled = "context canceled" -) - // token represents a request that is being processed. type token struct{} @@ -73,11 +68,13 @@ type token struct{ // // If the multiplier is <= 0, a noop middleware will be returned instead. // +// RetryAfter determines the Retry-After header value to be sent to throttled requests. +// // Useful links: // // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503 -func Throttle(cpuMultiplier int) gin.HandlerFunc { +func Throttle(cpuMultiplier int, retryAfter time.Duration) gin.HandlerFunc { if cpuMultiplier <= 0 { // throttling is disabled, return a noop middleware return func(c *gin.Context) {} @@ -89,36 +86,24 @@ func Throttle(cpuMultiplier int) gin.HandlerFunc { backlogChannelSize = limit + backlogLimit tokens = make(chan token, limit) backlogTokens = make(chan token, backlogChannelSize) - retryAfter = "30" // seconds - backlogDuration = 30 * time.Second + retryAfterStr = strconv.FormatUint(uint64(retryAfter/time.Second), 10) ) // prefill token channels for i := 0; i < limit; i++ { tokens <- token{} } - for i := 0; i < backlogChannelSize; i++ { backlogTokens <- token{} } - // bail instructs the requester to return after retryAfter seconds, returns a 503, - // and writes the given message into the "error" field of a returned json object - bail := func(c *gin.Context, msg string) { - c.Header("Retry-After", retryAfter) - c.JSON(http.StatusServiceUnavailable, gin.H{"error": msg}) - c.Abort() - } - return func(c *gin.Context) { // inside this select, the caller tries to get a backlog token select { case <-c.Request.Context().Done(): // request context has been canceled already - bail(c, errContextCanceled) + return case btok := <-backlogTokens: - // take a backlog token and wait - timer := time.NewTimer(backlogDuration) defer func() { // when we're finished, return the backlog token to the bucket backlogTokens <- btok @@ -127,16 +112,11 @@ func Throttle(cpuMultiplier int) gin.HandlerFunc { // inside *this* select, the caller has a backlog token, // and they're waiting for their turn to be processed select { - case <-timer.C: - // waiting too long in the backlog - bail(c, errTimedOut) case <-c.Request.Context().Done(): // the request context has been canceled already - timer.Stop() - bail(c, errContextCanceled) + return case tok := <-tokens: // the caller gets a token, so their request can now be processed - timer.Stop() defer func() { // whatever happens to the request, put the // token back in the bucket when we're finished @@ -147,7 +127,9 @@ func Throttle(cpuMultiplier int) gin.HandlerFunc { default: // we don't have space in the backlog queue - bail(c, errCapacityExceeded) + c.Header("Retry-After", retryAfterStr) + c.JSON(http.StatusTooManyRequests, gin.H{"error": "server capacity exceeded"}) + c.Abort() } } } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 4c1e890ef..8095e6612 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -27,6 +27,7 @@ "io" "net/http" "net/url" + "strconv" "strings" "sync" "time" @@ -84,36 +85,37 @@ type transport struct { } // GET will perform given http request using transport client, retrying on certain preset errors, or if status code is among retryOn. -func (t *transport) GET(r *http.Request, retryOn ...int) (*http.Response, error) { +func (t *transport) GET(r *http.Request) (*http.Response, error) { if r.Method != http.MethodGet { return nil, errors.New("must be GET request") } return t.do(r, func(r *http.Request) error { return t.signGET(r) - }, retryOn...) + }) } // POST will perform given http request using transport client, retrying on certain preset errors, or if status code is among retryOn. -func (t *transport) POST(r *http.Request, body []byte, retryOn ...int) (*http.Response, error) { +func (t *transport) POST(r *http.Request, body []byte) (*http.Response, error) { if r.Method != http.MethodPost { return nil, errors.New("must be POST request") } return t.do(r, func(r *http.Request) error { return t.signPOST(r, body) - }, retryOn...) + }) } -func (t *transport) do(r *http.Request, signer func(*http.Request) error, retryOn ...int) (*http.Response, error) { - const maxRetries = 5 +func (t *transport) do(r *http.Request, signer func(*http.Request) error) (*http.Response, error) { + const ( + // max no. attempts + maxRetries = 5 - var ( - // Initial backoff duration - backoff = 2 * time.Second - - // Get request hostname - host = r.URL.Hostname() + // starting backoff duration. + baseBackoff = 2 * time.Second ) + // Get request hostname + host := r.URL.Hostname() + // Check if recently reached max retries for this host // so we don't need to bother reattempting it. The only // errors that are retried upon are server failure and @@ -137,6 +139,8 @@ func (t *transport) do(r *http.Request, signer func(*http.Request) error, retryO r.Header.Set("User-Agent", t.controller.userAgent) for i := 0; i < maxRetries; i++ { + var backoff time.Duration + // Reset signing header fields now := t.controller.clock.Now().UTC() r.Header.Set("Date", now.Format("Mon, 02 Jan 2006 15:04:05")+" GMT") @@ -152,18 +156,35 @@ func (t *transport) do(r *http.Request, signer func(*http.Request) error, retryO // Attempt to perform request rsp, err := t.controller.client.Do(r) - if err == nil { //nolint shutup linter + if err == nil { //nolint:gocritic // TooManyRequest means we need to slow // down and retry our request. Codes over // 500 generally indicate temp. outages. if code := rsp.StatusCode; code < 500 && - code != http.StatusTooManyRequests && - !containsInt(retryOn, rsp.StatusCode) { + code != http.StatusTooManyRequests { return rsp, nil } // Generate error from status code for logging err = errors.New(`http response "` + rsp.Status + `"`) + + // Search for a provided "Retry-After" header value. + if after := rsp.Header.Get("Retry-After"); after != "" { + + if u, _ := strconv.ParseUint(after, 10, 32); u != 0 { + // An integer number of backoff seconds was provided. + backoff = time.Duration(u) * time.Second + } else if at, _ := http.ParseTime(after); !at.Before(now) { + // An HTTP formatted future date-time was provided. + backoff = at.Sub(now) + } + + // Don't let their provided backoff exceed our max. + if max := baseBackoff * maxRetries; backoff > max { + backoff = max + } + } + } else if errorsv2.Is(err, context.DeadlineExceeded, context.Canceled, @@ -179,11 +200,18 @@ func (t *transport) do(r *http.Request, signer func(*http.Request) error, retryO } else if errors.As(err, &x509.UnknownAuthorityError{}) { // Unknown authority errors we do NOT recover from return nil, err - } else if fastFail { + } + + if fastFail { // on fast-fail, don't bother backoff/retry return nil, fmt.Errorf("%w (fast fail)", err) } + if backoff == 0 { + // No retry-after found, set our predefined backoff. + backoff = time.Duration(i) * baseBackoff + } + l.Errorf("backing off for %s after http request error: %v", backoff.String(), err) select { @@ -238,13 +266,3 @@ func (t *transport) safesign(sign func()) { // Perform signing sign() } - -// containsInt checks if slice contains check. -func containsInt(slice []int, check int) bool { - for _, i := range slice { - if i == check { - return true - } - } - return false -} diff --git a/test/envparsing.sh b/test/envparsing.sh index c50fa99ea..bcf3811b9 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -83,6 +83,7 @@ GTS_SYSLOG_ADDRESS='127.0.0.1:6969' \ GTS_ADVANCED_COOKIES_SAMESITE='strict' \ GTS_ADVANCED_RATE_LIMIT_REQUESTS=6969 \ GTS_ADVANCED_THROTTLING_MULTIPLIER=-1 \ +GTS_ADVANCED_THROTTLING_RETRY_AFTER='10s' \ go run ./cmd/gotosocial/... --config-path internal/config/testdata/test.yaml debug config) OUTPUT_OUT=$(mktemp)