diff --git a/modules/caddyhttp/fileserver/browse.go b/modules/caddyhttp/fileserver/browse.go
index 0a623c79e..52aa7a9f8 100644
--- a/modules/caddyhttp/fileserver/browse.go
+++ b/modules/caddyhttp/fileserver/browse.go
@@ -130,9 +130,9 @@ func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w ht
 
 	// speed up browser/client experience and caching by supporting If-Modified-Since
 	if ifModSinceStr := r.Header.Get("If-Modified-Since"); ifModSinceStr != "" {
-		ifModSince, err := time.ParseInLocation(http.TimeFormat, ifModSinceStr, time.Local)
-		lastModTrunc := listing.lastModified.Truncate(time.Second)
-		if err == nil && (lastModTrunc.Equal(ifModSince) || lastModTrunc.Before(ifModSince)) {
+		// basically a copy of stdlib file server's handling of If-Modified-Since
+		ifModSince, err := http.ParseTime(ifModSinceStr)
+		if err == nil && listing.lastModified.Truncate(time.Second).Compare(ifModSince) <= 0 {
 			w.WriteHeader(http.StatusNotModified)
 			return nil
 		}
@@ -213,6 +213,11 @@ func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w ht
 }
 
 func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, fileSystem fs.FS, dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (*browseTemplateContext, error) {
+	// modTime for the directory itself
+	stat, err := dir.Stat()
+	if err != nil {
+		return nil, err
+	}
 	dirLimit := defaultDirEntryLimit
 	if fsrv.Browse.FileLimit != 0 {
 		dirLimit = fsrv.Browse.FileLimit
@@ -225,7 +230,7 @@ func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, fileSystem fs
 	// user can presumably browse "up" to parent folder if path is longer than "/"
 	canGoUp := len(urlPath) > 1
 
-	return fsrv.directoryListing(ctx, fileSystem, files, canGoUp, root, urlPath, repl), nil
+	return fsrv.directoryListing(ctx, fileSystem, stat.ModTime(), files, canGoUp, root, urlPath, repl), nil
 }
 
 // browseApplyQueryParams applies query parameters to the listing.
diff --git a/modules/caddyhttp/fileserver/browsetplcontext.go b/modules/caddyhttp/fileserver/browsetplcontext.go
index 8e5d138f1..b9489c6a6 100644
--- a/modules/caddyhttp/fileserver/browsetplcontext.go
+++ b/modules/caddyhttp/fileserver/browsetplcontext.go
@@ -35,15 +35,16 @@ import (
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 )
 
-func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) *browseTemplateContext {
+func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS, parentModTime time.Time, entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) *browseTemplateContext {
 	filesToHide := fsrv.transformHidePaths(repl)
 
 	name, _ := url.PathUnescape(urlPath)
 
 	tplCtx := &browseTemplateContext{
-		Name:    path.Base(name),
-		Path:    urlPath,
-		CanGoUp: canGoUp,
+		Name:         path.Base(name),
+		Path:         urlPath,
+		CanGoUp:      canGoUp,
+		lastModified: parentModTime,
 	}
 
 	for _, entry := range entries {
@@ -131,6 +132,10 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
 		})
 	}
 
+	// this time is used for the Last-Modified header and comparing If-Modified-Since from client
+	// both are expected to be in UTC, so we convert to UTC here
+	// see: https://github.com/caddyserver/caddy/issues/6828
+	tplCtx.lastModified = tplCtx.lastModified.UTC()
 	return tplCtx
 }