staticfiles: Support pre-compressed zstd, make etag content-encoding-aware (#2626)

* Add support for precompressed zstd files (rfc8478)

* Avoid the hash lookup for the file extension.

* Only calculate Etag once

caddyhttp/staticfiles/fileserver.go

@@ -184,15 +184,14 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request) (int, err
 		return http.StatusNotFound, nil
 	}
 
-	etag := calculateEtag(d)
+	etagInfo := d
-
 	// look for compressed versions of the file on disk, if the client supports that encoding
 	for _, encoding := range staticEncodingPriority {
 		// see if the client accepts a compressed encoding we offer
 		acceptEncoding := strings.Split(r.Header.Get("Accept-Encoding"), ",")
 		accepted := false
 		for _, acc := range acceptEncoding {
-			if strings.TrimSpace(acc) == encoding {
+			if strings.TrimSpace(acc) == encoding.name {
 				accepted = true
 				break
 			}
@@ -204,7 +203,7 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request) (int, err
 		}
 
 		// see if the compressed version of this file exists
-		encodedFile, err := fs.Root.Open(reqPath + staticEncoding[encoding])
+		encodedFile, err := fs.Root.Open(reqPath + encoding.ext)
 		if err != nil {
 			continue
 		}
@@ -222,13 +221,15 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request) (int, err
 
 		// the encoded file is now what we're serving
 		f = encodedFile
-		etag = calculateEtag(encodedFileInfo)
+		etagInfo = encodedFileInfo
 		w.Header().Add("Vary", "Accept-Encoding")
-		w.Header().Set("Content-Encoding", encoding)
+		w.Header().Set("Content-Encoding", encoding.name)
 		w.Header().Set("Content-Length", strconv.FormatInt(encodedFileInfo.Size(), 10))
 		break
 	}
 
+	etag := calculateEtag(etagInfo)
+
 	// Set the ETag returned to the user-agent. Note that a conditional If-None-Match
 	// request is handled in http.ServeContent below, which checks against this ETag value.
 	w.Header().Set("ETag", etag)
@@ -279,16 +280,9 @@ var DefaultIndexPages = []string{
 	"default.txt",
 }
 
-// staticEncoding is a map of content-encoding to a file extension.
-// If client accepts given encoding (via Accept-Encoding header) and compressed file with given extensions exists
-// it will be served to the client instead of original one.
-var staticEncoding = map[string]string{
-	"gzip": ".gz",
-	"br":   ".br",
-}
-
 // staticEncodingPriority is a list of preferred static encodings (most efficient compression to least one).
-var staticEncodingPriority = []string{
+var staticEncodingPriority = []struct{ name, ext string }{
-	"br",
+	{"zstd", ".zst"},
-	"gzip",
+	{"br", ".br"},
+	{"gzip", ".gz"},
 }

caddyhttp/staticfiles/fileserver_test.go

@@ -264,6 +264,17 @@ func TestServeHTTP(t *testing.T) {
 			expectedLocation:    "https://foo/example.com/../",
 			expectedBodyContent: movedPermanently,
 		},
+		// Test 30 - try to get pre- file.
+		{
+			url:                   "https://foo/sub/gzipped.html",
+			acceptEncoding:        "zstd",
+			expectedStatus:        http.StatusOK,
+			expectedBodyContent:   testFiles[webrootSubGzippedHTMLZst],
+			expectedEtag:          `"2n9ci"`,
+			expectedVary:          "Accept-Encoding",
+			expectedEncoding:      "zstd",
+			expectedContentLength: strconv.Itoa(len(testFiles[webrootSubGzippedHTMLZst])),
+		},
 	}
 
 	for i, test := range tests {
@@ -546,6 +557,7 @@ var (
 	webrootSubGzippedHTML              = filepath.Join(webrootName, "sub", "gzipped.html")
 	webrootSubGzippedHTMLGz            = filepath.Join(webrootName, "sub", "gzipped.html.gz")
 	webrootSubGzippedHTMLBr            = filepath.Join(webrootName, "sub", "gzipped.html.br")
+	webrootSubGzippedHTMLZst           = filepath.Join(webrootName, "sub", "gzipped.html.zst")
 	webrootSubBrotliHTML               = filepath.Join(webrootName, "sub", "brotli.html")
 	webrootSubBrotliHTMLGz             = filepath.Join(webrootName, "sub", "brotli.html.gz")
 	webrootSubBrotliHTMLBr             = filepath.Join(webrootName, "sub", "brotli.html.br")
@@ -573,9 +585,10 @@ var testFiles = map[string]string{
 	webrootSubGzippedHTML:              "<h1>gzipped.html</h1>",
 	webrootSubGzippedHTMLGz:            "1.gzipped.html.gz",
 	webrootSubGzippedHTMLBr:            "2.gzipped.html.br",
-	webrootSubBrotliHTML:               "3.brotli.html",
+	webrootSubGzippedHTMLZst:           "3.gzipped.html.zst",
-	webrootSubBrotliHTMLGz:             "4.brotli.html.gz",
+	webrootSubBrotliHTML:               "4.brotli.html",
-	webrootSubBrotliHTMLBr:             "5.brotli.html.br",
+	webrootSubBrotliHTMLGz:             "5.brotli.html.gz",
+	webrootSubBrotliHTMLBr:             "6.brotli.html.br",
 	webrootSubBarDirWithIndexIndexHTML: "<h1>bar/dirwithindex/index.html</h1>",
 }