[feature] Object store custom URL (S3) (#3046)

* tweaks

* boobs

* fix variable name + typo

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

docs/configuration/storage.md

@@ -30,11 +30,42 @@ storage-local-base-path: "/gotosocial/storage"
 # Default: ""
 storage-s3-endpoint: ""
 
-# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
+# Bool. Set this to true if data stored in S3 should be proxied through
+# GoToSocial instead of forwarding the request to a presigned URL.
+#
+# In most cases you won't need to touch this setting, but it might be useful
+# if it's not possible for your bucket provider to generate presigned URLs,
+# or if your bucket is not able to exposed to the wider internet.
 #
 # Default: false
 storage-s3-proxy: false
 
+# String. URL to use a base for redirecting incoming media requests to.
+#
+# Must start with "http://" or "https://" and end without a trailing slash.
+#
+# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for
+# "normal" s3 usage, and most admins can happily just ignore this setting.
+#
+# If set, then media fileserver requests to your instance will be redirected
+# to this URL instead of your bucket URL, preserving relevant path parts.
+#
+# This is useful if you are using a CDN proxy in front of your S3 bucket, and you
+# want to serve media from the CDN rather than serving from your S3 bucket directly.
+#
+# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org",
+# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org",
+# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org".
+#
+# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org",
+# but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
+#
+# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true.
+#
+# Examples: ["https://cdn.some-fancy-host.org"]
+# Default: ""
+storage-s3-redirect-url: ""
+
 # Bool. Use SSL for S3 connections.
 #
 # Only set this to 'false' when testing locally.
@@ -76,7 +107,7 @@ storage-s3-bucket: ""
 GoToSocial by default creates signed URL's which means we don't need to change anything major on the policies of the bucket.
 
 1. Login to AWS -> select S3 as service.
-2. click Create Bucket
+2. Click Create Bucket
 3. Provide a unique name and avoid adding "." in the name
 4. Do not change the public access settings (Let them be on "block public access" mode)
 
@@ -110,6 +141,14 @@ GoToSocial by default creates signed URL's which means we don't need to change a
     * `storage-s3-secret-key` -> Secret key you obtained for the user created above
     * `storage-s3-bucket` -> The `<bucketname>` that you created just now
 
+### `storage-s3-redirect-url`
+
+If you are using a CDN in front of your S3 bucket, and you want to serve media from the CDN rather than serving from your S3 bucket directly, you should set the `storage-s3-redirect-url` to the CDN URL.
+
+For example, if you have your `storage-s3-endpoint` value set to "s3.my-storage.example.org", and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org", then you should set `storage-s3-redirect-url` to "https://cdn.some-fancy-host.org".
+
+This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org", but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
+
 ## Storage migration
 
 Migration between backends is freely possible. To do so, you only have to move the directories (and their contents) between the different implementations.

example/config.yaml

@@ -551,11 +551,42 @@ storage-local-base-path: "/gotosocial/storage"
 # Default: ""
 storage-s3-endpoint: ""
 
-# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
+# Bool. Set this to true if data stored in S3 should be proxied through
+# GoToSocial instead of forwarding the request to a presigned URL.
+#
+# In most cases you won't need to touch this setting, but it might be useful
+# if it's not possible for your bucket provider to generate presigned URLs,
+# or if your bucket is not able to exposed to the wider internet.
 #
 # Default: false
 storage-s3-proxy: false
 
+# String. URL to use a base for redirecting incoming media requests to.
+#
+# Must start with "http://" or "https://" and end without a trailing slash.
+#
+# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for
+# "normal" s3 usage, and most admins can happily just ignore this setting.
+#
+# If set, then media fileserver requests to your instance will be redirected
+# to this URL instead of your bucket URL, preserving relevant path parts.
+#
+# This is useful if you are using a CDN proxy in front of your S3 bucket, and you
+# want to serve media from the CDN rather than serving from your S3 bucket directly.
+#
+# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org",
+# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org",
+# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org".
+#
+# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org",
+# but direct callers to *download* that data from "https://cdn.some-fancy-host.org".
+#
+# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true.
+#
+# Examples: ["https://cdn.some-fancy-host.org"]
+# Default: ""
+storage-s3-redirect-url: ""
+
 # Bool. Use SSL for S3 connections.
 #
 # Only set this to 'false' when testing locally.

internal/config/config.go

@@ -110,6 +110,7 @@ type Configuration struct {
 	StorageS3UseSSL      bool   `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
 	StorageS3BucketName  string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
 	StorageS3Proxy       bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
+	StorageS3RedirectURL string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."`
 
 	StatusesMaxChars           int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"`
 	StatusesPollMaxOptions     int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`

internal/config/defaults.go

@@ -85,6 +85,7 @@
 	StorageLocalBasePath: "/gotosocial/storage",
 	StorageS3UseSSL:      true,
 	StorageS3Proxy:       false,
+	StorageS3RedirectURL: "",
 
 	StatusesMaxChars:           5000,
 	StatusesPollMaxOptions:     6,

internal/config/helpers.gen.go

@@ -1500,6 +1500,31 @@ func GetStorageS3Proxy() bool { return global.GetStorageS3Proxy() }
 // SetStorageS3Proxy safely sets the value for global configuration 'StorageS3Proxy' field
 func SetStorageS3Proxy(v bool) { global.SetStorageS3Proxy(v) }
 
+// GetStorageS3RedirectURL safely fetches the Configuration value for state's 'StorageS3RedirectURL' field
+func (st *ConfigState) GetStorageS3RedirectURL() (v string) {
+	st.mutex.RLock()
+	v = st.config.StorageS3RedirectURL
+	st.mutex.RUnlock()
+	return
+}
+
+// SetStorageS3RedirectURL safely sets the Configuration value for state's 'StorageS3RedirectURL' field
+func (st *ConfigState) SetStorageS3RedirectURL(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.StorageS3RedirectURL = v
+	st.reloadToViper()
+}
+
+// StorageS3RedirectURLFlag returns the flag name for the 'StorageS3RedirectURL' field
+func StorageS3RedirectURLFlag() string { return "storage-s3-redirect-url" }
+
+// GetStorageS3RedirectURL safely fetches the value for global configuration 'StorageS3RedirectURL' field
+func GetStorageS3RedirectURL() string { return global.GetStorageS3RedirectURL() }
+
+// SetStorageS3RedirectURL safely sets the value for global configuration 'StorageS3RedirectURL' field
+func SetStorageS3RedirectURL(v string) { global.SetStorageS3RedirectURL(v) }
+
 // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
 func (st *ConfigState) GetStatusesMaxChars() (v int) {
 	st.mutex.RLock()

internal/config/validate.go

@@ -19,6 +19,8 @@
 
 import (
 	"fmt"
+	"net/url"
+	"strings"
 
 	"github.com/miekg/dns"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -118,6 +120,28 @@ func Validate() error {
 		errf("%s must be set", WebAssetBaseDirFlag())
 	}
 
+	// `storage-s3-redirect-url`
+	if s3RedirectURL := GetStorageS3RedirectURL(); s3RedirectURL != "" {
+		if strings.HasSuffix(s3RedirectURL, "/") {
+			errf(
+				"%s must not end with a trailing slash",
+				StorageS3RedirectURLFlag(),
+			)
+		}
+
+		if url, err := url.Parse(s3RedirectURL); err != nil {
+			errf(
+				"%s invalid: %w",
+				StorageS3RedirectURLFlag(), err,
+			)
+		} else if url.Scheme != "https" && url.Scheme != "http" {
+			errf(
+				"%s scheme must be https or http",
+				StorageS3RedirectURLFlag(),
+			)
+		}
+	}
+
 	// Custom / LE TLS settings.
 	//
 	// Only one of custom certs or LE can be set,

internal/storage/storage.go

@@ -79,6 +79,7 @@ type Driver struct {
 	Proxy          bool
 	Bucket         string
 	PresignedCache *ttl.Cache[string, PresignedURL]
+	RedirectURL    string
 }
 
 // Get returns the byte value for key in storage.
@@ -163,12 +164,27 @@ func (d *Driver) URL(ctx context.Context, key string) *PresignedURL {
 		return &e.Value
 	}
 
-	u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{
+	var (
-		"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
+		u   *url.URL
-	})
+		err error
-	if err != nil {
+	)
-		// If URL request fails, fallback is to fetch the file. So ignore the error here
+
-		return nil
+	if d.RedirectURL != "" {
+		u, err = url.Parse(d.RedirectURL + "/" + key)
+		if err != nil {
+			// If URL parsing fails, fallback is to
+			// fetch the file. So ignore the error here
+			return nil
+		}
+	} else {
+		u, err = s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{
+			"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
+		})
+		if err != nil {
+			// If URL request fails, fallback is to
+			// fetch the file. So ignore the error here
+			return nil
+		}
 	}
 
 	psu := PresignedURL{
@@ -204,6 +220,14 @@ func (d *Driver) ProbeCSPUri(ctx context.Context) (string, error) {
 		return "", nil
 	}
 
+	// If an S3 redirect URL is set, just
+	// return this URL without probing; we
+	// likely don't have write access on it
+	// anyway since it's probs a CDN bucket.
+	if d.RedirectURL != "" {
+		return d.RedirectURL + "/", nil
+	}
+
 	const cspKey = "gotosocial-csp-probe"
 
 	// Create an empty file in S3 storage.
@@ -273,6 +297,7 @@ func NewS3Storage() (*Driver, error) {
 	secret := config.GetStorageS3SecretKey()
 	secure := config.GetStorageS3UseSSL()
 	bucket := config.GetStorageS3BucketName()
+	redirectURL := config.GetStorageS3RedirectURL()
 
 	// Open the s3 storage implementation
 	s3, err := s3.Open(endpoint, bucket, &s3.Config{
@@ -300,5 +325,6 @@ func NewS3Storage() (*Driver, error) {
 		Bucket:         config.GetStorageS3BucketName(),
 		Storage:        s3,
 		PresignedCache: presignedCache,
+		RedirectURL:    redirectURL,
 	}, nil
 }

test/envparsing.sh

@@ -173,6 +173,7 @@ EXPECT=$(cat << "EOF"
     "storage-s3-bucket": "gts",
     "storage-s3-endpoint": "localhost:9000",
     "storage-s3-proxy": true,
+    "storage-s3-redirect-url": "",
     "storage-s3-secret-key": "miniostorage",
     "storage-s3-use-ssl": false,
     "syslog-address": "127.0.0.1:6969",
@@ -253,6 +254,7 @@ GTS_STORAGE_S3_SECRET_KEY='miniostorage' \
 GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
 GTS_STORAGE_S3_USE_SSL='false' \
 GTS_STORAGE_S3_PROXY='true' \
+GTS_STORAGE_S3_REDIRECT_URL='' \
 GTS_STORAGE_S3_BUCKET='gts' \
 GTS_STATUSES_MAX_CHARS=69 \
 GTS_STATUSES_CW_MAX_CHARS=420 \