feat: allow fs embedding with --embed (#160)

* feat: allow fs embedding with `--embed`

* code review

* remove utils/io.go

* don't shadow `err`

* add syntax and examples to README

README.md

@@ -62,6 +62,7 @@ Syntax:
 $ xcaddy build [<caddy_version>]
     [--output <file>]
     [--with <module[@version][=replacement]>...]
+    [--embed <[alias]:path/to/dir>...]
 ```
 
 - `<caddy_version>` is the core Caddy version to build; defaults to `CADDY_VERSION` env variable or latest.<br>
@@ -72,6 +73,7 @@ $ xcaddy build [<caddy_version>]
 
 - `--output` changes the output file.
 - `--with` can be used multiple times to add plugins by specifying the Go module name and optionally its version, similar to `go get`. Module name is required, but specific version and/or local replacement are optional.
+- `--embed` can be used multiple times to embed directories into the built Caddy executable. The directory can be prefixed with a custom alias and a colon `:` to use it with the `root` directive and sub-directive. 
 
 Examples:
 
@@ -107,6 +109,24 @@ $ xcaddy build \
 
 This allows you to hack on Caddy core (and optionally plug in extra modules at the same time!) with relative ease.
 
+```
+$ xcaddy build --embed foo:./sites/foo --embed bar:./sites/bar
+$ cat Caddyfile
+foo.localhost {
+	root * /foo
+	file_server {
+		fs embedded
+	}
+}
+
+bar.localhost {
+	root * /bar
+	file_server {
+		fs embedded
+	}
+}
+```
+This allows you to serve 2 sites from 2 different embedded directories, which are referenced by aliases, from a single Caddy executable.
 
 ### For plugin development
 

builder.go

@@ -45,6 +45,12 @@ type Builder struct {
 	Debug        bool          `json:"debug,omitempty"`
 	BuildFlags   string        `json:"build_flags,omitempty"`
 	ModFlags     string        `json:"mod_flags,omitempty"`
+
+	// Experimental: subject to change
+	EmbedDirs []struct {
+		Dir  string `json:"dir,omitempty"`
+		Name string `json:"name,omitempty"`
+	} `json:"embed_dir,omitempty"`
 }
 
 // Build builds Caddy at the configured version with the
@@ -66,6 +72,7 @@ func (b Builder) Build(ctx context.Context, outputFile string) error {
 	if err != nil {
 		return err
 	}
+	log.Printf("[INFO] absolute output file path: %s", absOutputFile)
 
 	// set some defaults from the environment, if applicable
 	if b.OS == "" {

cmd/main.go

@@ -71,6 +71,7 @@ func runBuild(ctx context.Context, args []string) error {
 	var argCaddyVersion, output string
 	var plugins []xcaddy.Dependency
 	var replacements []xcaddy.Replace
+	var embedDir []string
 	for i := 0; i < len(args); i++ {
 		switch args[i] {
 		case "--with":
@@ -105,7 +106,12 @@ func runBuild(ctx context.Context, args []string) error {
 			}
 			i++
 			output = args[i]
-
+		case "--embed":
+			if i == len(args)-1 {
+				return fmt.Errorf("expected value after --embed flag")
+			}
+			i++
+			embedDir = append(embedDir, args[i])
 		default:
 			if argCaddyVersion != "" {
 				return fmt.Errorf("missing flag; caddy version already set at %s", argCaddyVersion)
@@ -139,6 +145,23 @@ func runBuild(ctx context.Context, args []string) error {
 		BuildFlags:   buildFlags,
 		ModFlags:     modFlags,
 	}
+	for _, md := range embedDir {
+		if before, after, found := strings.Cut(md, ":"); found {
+			builder.EmbedDirs = append(builder.EmbedDirs, struct {
+				Dir  string `json:"dir,omitempty"`
+				Name string `json:"name,omitempty"`
+			}{
+				after, before,
+			})
+		} else {
+			builder.EmbedDirs = append(builder.EmbedDirs, struct {
+				Dir  string `json:"dir,omitempty"`
+				Name string `json:"name,omitempty"`
+			}{
+				before, "",
+			})
+		}
+	}
 	err := builder.Build(ctx, output)
 	if err != nil {
 		log.Fatalf("[FATAL] %v", err)

environment.go

@@ -86,11 +86,40 @@ func (b Builder) newEnvironment(ctx context.Context) (*environment, error) {
 	// write the main module file to temporary folder
 	mainPath := filepath.Join(tempFolder, "main.go")
 	log.Printf("[INFO] Writing main module: %s\n%s", mainPath, buf.Bytes())
-	err = os.WriteFile(mainPath, buf.Bytes(), 0644)
+	err = os.WriteFile(mainPath, buf.Bytes(), 0o644)
 	if err != nil {
 		return nil, err
 	}
 
+	if len(b.EmbedDirs) > 0 {
+		for _, d := range b.EmbedDirs {
+			err = copy(d.Dir, filepath.Join(tempFolder, "files", d.Name))
+			if err != nil {
+				return nil, err
+			}
+			_, err = os.Stat(d.Dir)
+			if err != nil {
+				return nil, fmt.Errorf("embed directory does not exist: %s", d.Dir)
+			}
+			log.Printf("[INFO] Embedding directory: %s", d.Dir)
+			buf.Reset()
+			tpl, err = template.New("embed").Parse(embeddedModuleTemplate)
+			if err != nil {
+				return nil, err
+			}
+			err = tpl.Execute(&buf, tplCtx)
+			if err != nil {
+				return nil, err
+			}
+			log.Printf("[INFO] Writing 'embedded' module: %s\n%s", mainPath, buf.Bytes())
+			emedPath := filepath.Join(tempFolder, "embed.go")
+			err = os.WriteFile(emedPath, buf.Bytes(), 0o644)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	env := &environment{
 		caddyVersion:    b.CaddyVersion,
 		plugins:         b.Plugins,
@@ -337,3 +366,113 @@ func main() {
 	caddycmd.Main()
 }
 `
+
+// originally published in: https://github.com/mholt/caddy-embed
+const embeddedModuleTemplate = `package main
+
+import (
+	"embed"
+	"io/fs"
+	"strings"
+
+	"{{.CaddyModule}}"
+	"{{.CaddyModule}}/caddyconfig/caddyfile"
+)
+
+// embedded is what will contain your static files. The go command
+// will automatically embed the files subfolder into this virtual
+// file system. You can optionally change the go:embed directive
+// to embed other files or folders.
+//
+//go:embed files
+var embedded embed.FS
+
+// files is the actual, more generic file system to be utilized.
+var files fs.FS = embedded
+
+// topFolder is the name of the top folder of the virtual
+// file system. go:embed does not let us add the contents
+// of a folder to the root of a virtual file system, so
+// if we want to trim that root folder prefix, we need to
+// also specify it in code as a string. Otherwise the
+// user would need to add configuration or code to trim
+// this root prefix from all filenames, e.g. specifying
+// "root files" in their file_server config.
+//
+// It is NOT REQUIRED to change this if changing the
+// go:embed directive; it is just for convenience in
+// the default case.
+const topFolder = "files"
+
+func init() {
+	caddy.RegisterModule(FS{})
+	stripFolderPrefix()
+}
+
+// stripFolderPrefix opens the root of the file system. If it
+// contains only 1 file, being a directory with the same
+// name as the topFolder const, then the file system will
+// be fs.Sub()'ed so the contents of the top folder can be
+// accessed as if they were in the root of the file system.
+// This is a convenience so most users don't have to add
+// additional configuration or prefix their filenames
+// unnecessarily.
+func stripFolderPrefix() error {
+	if f, err := files.Open("."); err == nil {
+		defer f.Close()
+
+		if dir, ok := f.(fs.ReadDirFile); ok {
+			entries, err := dir.ReadDir(2)
+			if err == nil &&
+				len(entries) == 1 &&
+				entries[0].IsDir() &&
+				entries[0].Name() == topFolder {
+				if sub, err := fs.Sub(embedded, topFolder); err == nil {
+					files = sub
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// FS implements a Caddy module and fs.FS for an embedded
+// file system provided by an unexported package variable.
+//
+// To use, simply put your files in a subfolder called
+// "files", then build Caddy with your local copy of this
+// plugin. Your site's files will be embedded directly
+// into the binary.
+//
+// If the embedded file system contains only one file in
+// its root which is a folder named "files", this module
+// will strip that folder prefix using fs.Sub(), so that
+// the contents of the folder can be accessed by name as
+// if they were in the actual root of the file system.
+// In other words, before: files/foo.txt, after: foo.txt.
+type FS struct{}
+
+// CaddyModule returns the Caddy module information.
+func (FS) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "caddy.fs.embedded",
+		New: func() caddy.Module { return new(FS) },
+	}
+}
+
+func (FS) Open(name string) (fs.File, error) {
+	// TODO: the file server doesn't clean up leading and trailing slashes, but embed.FS is particular so we remove them here; I wonder if the file server should be tidy in the first place
+	name = strings.Trim(name, "/")
+	return files.Open(name)
+}
+
+// UnmarshalCaddyfile exists so this module can be used in
+// the Caddyfile, but there is nothing to unmarshal.
+func (FS) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil }
+
+// Interface guards
+var (
+	_ fs.FS                 = (*FS)(nil)
+	_ caddyfile.Unmarshaler = (*FS)(nil)
+)
+`

go.mod

@@ -3,6 +3,6 @@ module github.com/caddyserver/xcaddy
 go 1.14
 
 require (
-	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Masterminds/semver/v3 v3.2.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 )

go.sum

@@ -1,4 +1,4 @@
-github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
+github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
-github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=

io.go

@@ -0,0 +1,64 @@
+package xcaddy
+
+// credit: https://github.com/goreleaser/goreleaser/blob/3f54b5eb2f13e86f07420124818fb6594f966278/internal/gio/copy.go
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// copy recursively copies src into dst with src's file modes.
+func copy(src, dst string) error {
+	src = filepath.ToSlash(src)
+	dst = filepath.ToSlash(dst)
+	log.Printf("[INFO] copying files: src=%s dest=%s", src, dst)
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return fmt.Errorf("failed to copy %s to %s: %w", src, dst, err)
+		}
+		path = filepath.ToSlash(path)
+		// We have the following:
+		// - src = "a/b"
+		// - dst = "dist/linuxamd64/b"
+		// - path = "a/b/c.txt"
+		// So we join "a/b" with "c.txt" and use it as the destination.
+		dst := filepath.ToSlash(filepath.Join(dst, strings.Replace(path, src, "", 1)))
+		if info.IsDir() {
+			return os.MkdirAll(dst, info.Mode())
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return copySymlink(path, dst)
+		}
+		return copyFile(path, dst, info.Mode())
+	})
+}
+
+func copySymlink(src, dst string) error {
+	src, err := os.Readlink(src)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(src, dst)
+}
+
+func copyFile(src, dst string, mode os.FileMode) error {
+	original, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open '%s': %w", src, err)
+	}
+	defer original.Close()
+
+	f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+	if err != nil {
+		return fmt.Errorf("failed to open '%s': %w", dst, err)
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, original); err != nil {
+		return fmt.Errorf("failed to copy: %w", err)
+	}
+	return nil
+}