Handle replacements robustly (#69)

* Parse JSON output of 'go list'

It's more robust in this way

* Handle all possible replacements properly for runDev()

1.  Handle all possible replacements properly for runDev()

    Replacement targets are not always paths.

    Reference:
      https://pkg.go.dev/cmd/go/internal/list#pkg-variables

2.  Parse replacement info from 'go list -m -json all'

    It's more robust in this way

* Test the 'go list -m -json all' parsing logic

* Call 'go list' command only once

`go list -m -json all` contains all informations we need so we don't
need extra `go list -m -json` call.

* main_windows_test.go: Make tests for Windows

* Support Go 1.16

* Use struct instead of map[string]interface{}

* extract and unexport the `module` struct

Co-authored-by: Mohammed Al Sahaf <msaa1990@gmail.com>

cmd/main.go

@@ -15,8 +15,11 @@
 package xcaddycmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -165,53 +168,24 @@ func getCaddyOutputFile() string {
 func runDev(ctx context.Context, args []string) error {
 	binOutput := getCaddyOutputFile()
 
-	// get current/main module name
+	// get current/main module name and the root directory of the main module
-	cmd := exec.Command("go", "list", "-m")
+	//
+	// make sure the module being developed is replaced
+	// so that the local copy is used
+	//
+	// replace directives only apply to the top-level/main go.mod,
+	// and since this tool is a carry-through for the user's actual
+	// go.mod, we need to transfer their replace directives through
+	// to the one we're making
+	cmd := exec.Command("go", "list", "-m", "-json", "all")
 	cmd.Stderr = os.Stderr
 	out, err := cmd.Output()
 	if err != nil {
 		return fmt.Errorf("exec %v: %v: %s", cmd.Args, err, string(out))
 	}
-	currentModule := strings.TrimSpace(string(out))
+	currentModule, moduleDir, replacements, err := parseGoListJson(out)
-
-	// get the root directory of the main module
-	cmd = exec.Command("go", "list", "-m", "-f={{.Dir}}")
-	cmd.Stderr = os.Stderr
-	out, err = cmd.Output()
 	if err != nil {
-		return fmt.Errorf("exec %v: %v: %s", cmd.Args, err, string(out))
+		return fmt.Errorf("json parse error: %v", err)
-	}
-	moduleDir := strings.TrimSpace(string(out))
-
-	// make sure the module being developed is replaced
-	// so that the local copy is used
-	replacements := []xcaddy.Replace{
-		xcaddy.NewReplace(currentModule, moduleDir),
-	}
-
-	// replace directives only apply to the top-level/main go.mod,
-	// and since this tool is a carry-through for the user's actual
-	// go.mod, we need to transfer their replace directives through
-	// to the one we're making
-	cmd = exec.Command("go", "list", "-m", "-f={{if .Replace}}{{.Path}}=>{{.Replace}}{{end}}", "all")
-	cmd.Stderr = os.Stderr
-	out, err = cmd.Output()
-	if err != nil {
-		return fmt.Errorf("exec %v: %v: %s", cmd.Args, err, string(out))
-	}
-	for _, line := range strings.Split(string(out), "\n") {
-		parts := strings.Split(line, "=>")
-		if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
-			continue
-		}
-
-		// adjust relative replacements in original module since our temporary module is in a different directory
-		if !filepath.IsAbs(parts[1]) {
-			parts[1] = filepath.Join(moduleDir, parts[1])
-			log.Printf("[INFO] Resolved relative replacement %s to %s", line, parts[1])
-		}
-
-		replacements = append(replacements, xcaddy.NewReplace(parts[0], parts[1]))
 	}
 
 	// reconcile remaining path segments; for example if a module foo/a
@@ -277,6 +251,75 @@ func runDev(ctx context.Context, args []string) error {
 	return cmd.Wait()
 }
 
+type module struct {
+	Path    string  // module path
+	Version string  // module version
+	Replace *module // replaced by this module
+	Main    bool    // is this the main module?
+	Dir     string  // directory holding files for this module, if any
+}
+
+func parseGoListJson(out []byte) (currentModule, moduleDir string, replacements []xcaddy.Replace, err error) {
+	var unjoinedReplaces []int
+
+	decoder := json.NewDecoder(bytes.NewReader(out))
+	for {
+		var mod module
+		if err = decoder.Decode(&mod); err == io.EOF {
+			err = nil
+			break
+		} else if err != nil {
+			return
+		}
+
+		if mod.Main {
+			// Current module is main module, retrieve the main module name and
+			// root directory path of the main module
+			currentModule = mod.Path
+			moduleDir = mod.Dir
+			replacements = append(replacements, xcaddy.NewReplace(currentModule, moduleDir))
+			continue
+		}
+
+		// Skip if current module is not replacement
+		if mod.Replace == nil {
+			continue
+		}
+
+		src := mod.Path + "@" + mod.Version
+
+		// 1. Target is module, version is required in this case
+		// 2A. Target is absolute path
+		// 2B. Target is relative path, proper handling is required in this case
+		dstPath := mod.Replace.Path
+		dstVersion := mod.Replace.Version
+		var dst string
+		if dstVersion != "" {
+			dst = dstPath + "@" + dstVersion
+		} else if filepath.IsAbs(dstPath) {
+			dst = dstPath
+		} else {
+			if moduleDir != "" {
+				dst = filepath.Join(moduleDir, dstPath)
+				log.Printf("[INFO] Resolved relative replacement %s to %s", dstPath, dst)
+			} else {
+				// moduleDir is not parsed yet, defer to later
+				dst = dstPath
+				unjoinedReplaces = append(unjoinedReplaces, len(replacements))
+			}
+		}
+
+		replacements = append(replacements, xcaddy.NewReplace(src, dst))
+	}
+	for _, idx := range unjoinedReplaces {
+		unresolved := string(replacements[idx].New)
+		resolved := filepath.Join(moduleDir, unresolved)
+		log.Printf("[INFO] Resolved relative replacement %s to %s", unresolved, resolved)
+		replacements[idx].New = xcaddy.ReplacementPath(resolved)
+	}
+	return
+}
+
 func normalizeImportPath(currentModule, cwd, moduleDir string) string {
 	return path.Join(currentModule, filepath.ToSlash(strings.TrimPrefix(cwd, moduleDir)))
 }

cmd/main_unix_test.go

@@ -0,0 +1,108 @@
+//go:build !windows
+// +build !windows
+
+package xcaddycmd
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/caddyserver/xcaddy"
+)
+
+func TestParseGoListJson(t *testing.T) {
+	currentModule, moduleDir, replacements, err := parseGoListJson([]byte(`
+{
+	"Path": "replacetest1",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "golang.org/x/example",
+		"Version": "v0.0.0-20210811190340-787a929d5a0d",
+		"Time": "2021-08-11T19:03:40Z",
+		"GoMod": "/home/simnalamburt/.go/pkg/mod/cache/download/golang.org/x/example/@v/v0.0.0-20210811190340-787a929d5a0d.mod",
+		"GoVersion": "1.15"
+	},
+	"GoMod": "/home/simnalamburt/.go/pkg/mod/cache/download/golang.org/x/example/@v/v0.0.0-20210811190340-787a929d5a0d.mod",
+	"GoVersion": "1.15"
+}
+{
+	"Path": "replacetest2",
+	"Version": "v0.0.1",
+	"Replace": {
+		"Path": "golang.org/x/example",
+		"Version": "v0.0.0-20210407023211-09c3a5e06b5d",
+		"Time": "2021-04-07T02:32:11Z",
+		"GoMod": "/home/simnalamburt/.go/pkg/mod/cache/download/golang.org/x/example/@v/v0.0.0-20210407023211-09c3a5e06b5d.mod",
+		"GoVersion": "1.15"
+	},
+	"GoMod": "/home/simnalamburt/.go/pkg/mod/cache/download/golang.org/x/example/@v/v0.0.0-20210407023211-09c3a5e06b5d.mod",
+	"GoVersion": "1.15"
+}
+{
+	"Path": "replacetest3",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "./fork1",
+		"Dir": "/home/work/module/fork1",
+		"GoMod": "/home/work/module/fork1/go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "/home/work/module/fork1",
+	"GoMod": "/home/work/module/fork1/go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "github.com/simnalamburt/module",
+	"Main": true,
+	"Dir": "/home/work/module",
+	"GoMod": "/home/work/module/go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "replacetest4",
+	"Version": "v0.0.1",
+	"Replace": {
+		"Path": "/srv/fork2",
+		"Dir": "/home/work/module/fork2",
+		"GoMod": "/home/work/module/fork2/go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "/home/work/module/fork2",
+	"GoMod": "/home/work/module/fork2/go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "replacetest5",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "./fork3",
+		"Dir": "/home/work/module/fork3",
+		"GoMod": "/home/work/module/fork3/go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "/home/work/module/fork3",
+	"GoMod": "/home/work/module/fork3/go.mod",
+	"GoVersion": "1.17"
+}
+`))
+	if err != nil {
+		t.Errorf("Error occured during JSON parsing")
+	}
+	if currentModule != "github.com/simnalamburt/module" {
+		t.Errorf("Unexpected module name")
+	}
+	if moduleDir != "/home/work/module" {
+		t.Errorf("Unexpected module path")
+	}
+	expected := []xcaddy.Replace{
+		xcaddy.NewReplace("replacetest1@v1.2.3", "golang.org/x/example@v0.0.0-20210811190340-787a929d5a0d"),
+		xcaddy.NewReplace("replacetest2@v0.0.1", "golang.org/x/example@v0.0.0-20210407023211-09c3a5e06b5d"),
+		xcaddy.NewReplace("replacetest3@v1.2.3", "/home/work/module/fork1"),
+		xcaddy.NewReplace("github.com/simnalamburt/module", "/home/work/module"),
+		xcaddy.NewReplace("replacetest4@v0.0.1", "/srv/fork2"),
+		xcaddy.NewReplace("replacetest5@v1.2.3", "/home/work/module/fork3"),
+	}
+	if !reflect.DeepEqual(replacements, expected) {
+		t.Errorf("Expected replacements '%v' but got '%v'", expected, replacements)
+	}
+}

cmd/main_windows_test.go

@@ -0,0 +1,112 @@
+//go:build windows
+// +build windows
+
+package xcaddycmd
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/caddyserver/xcaddy"
+)
+
+func TestParseGoListJson(t *testing.T) {
+	currentModule, moduleDir, replacements, err := parseGoListJson([]byte(`
+{
+	"Path": "replacetest1",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "golang.org/x/example",
+		"Version": "v0.0.0-20210811190340-787a929d5a0d",
+		"Time": "2021-08-11T19:03:40Z",
+		"Dir": "C:\\Users\\simna\\go\\pkg\\mod\\golang.org\\x\\example@v0.0.0-20210811190340-787a929d5a0d",
+		"GoMod": "C:\\Users\\simna\\go\\pkg\\mod\\cache\\download\\golang.org\\x\\example\\@v\\v0.0.0-20210811190340-787a929d5a0d.mod",
+		"GoVersion": "1.15"
+	},
+	"Dir": "C:\\Users\\simna\\go\\pkg\\mod\\golang.org\\x\\example@v0.0.0-20210811190340-787a929d5a0d",
+	"GoMod": "C:\\Users\\simna\\go\\pkg\\mod\\cache\\download\\golang.org\\x\\example\\@v\\v0.0.0-20210811190340-787a929d5a0d.mod",
+	"GoVersion": "1.15"
+}
+{
+	"Path": "replacetest2",
+	"Version": "v0.0.1",
+	"Replace": {
+		"Path": "golang.org/x/example",
+		"Version": "v0.0.0-20210407023211-09c3a5e06b5d",
+		"Time": "2021-04-07T02:32:11Z",
+		"Dir": "C:\\Users\\simna\\go\\pkg\\mod\\golang.org\\x\\example@v0.0.0-20210407023211-09c3a5e06b5d",
+		"GoMod": "C:\\Users\\simna\\go\\pkg\\mod\\cache\\download\\golang.org\\x\\example\\@v\\v0.0.0-20210407023211-09c3a5e06b5d.mod",
+		"GoVersion": "1.15"
+	},
+	"Dir": "C:\\Users\\simna\\go\\pkg\\mod\\golang.org\\x\\example@v0.0.0-20210407023211-09c3a5e06b5d",
+	"GoMod": "C:\\Users\\simna\\go\\pkg\\mod\\cache\\download\\golang.org\\x\\example\\@v\\v0.0.0-20210407023211-09c3a5e06b5d.mod",
+	"GoVersion": "1.15"
+}
+{
+	"Path": "replacetest3",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "./fork1",
+		"Dir": "C:\\Users\\work\\module\\fork1",
+		"GoMod": "C:\\Users\\work\\module\\fork1\\go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "C:\\Users\\work\\module\\fork1",
+	"GoMod": "C:\\Users\\work\\module\\fork1\\go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "github.com/simnalamburt/module",
+	"Main": true,
+	"Dir": "C:\\Users\\work\\module",
+	"GoMod": "C:\\Users\\work\\module\\go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "replacetest4",
+	"Version": "v0.0.1",
+	"Replace": {
+		"Path": "C:\\go\\fork2",
+		"Dir": "C:\\Users\\work\\module\\fork2",
+		"GoMod": "C:\\Users\\work\\module\\fork2\\go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "C:\\Users\\work\\module\\fork2",
+	"GoMod": "C:\\Users\\work\\module\\fork2\\go.mod",
+	"GoVersion": "1.17"
+}
+{
+	"Path": "replacetest5",
+	"Version": "v1.2.3",
+	"Replace": {
+		"Path": "./fork3",
+		"Dir": "C:\\Users\\work\\module\\fork3",
+		"GoMod": "C:\\Users\\work\\module\\fork1\\go.mod",
+		"GoVersion": "1.17"
+	},
+	"Dir": "C:\\Users\\work\\module\\fork3",
+	"GoMod": "C:\\Users\\work\\module\\fork3\\go.mod",
+	"GoVersion": "1.17"
+}
+`))
+	if err != nil {
+		t.Errorf("Error occured during JSON parsing")
+	}
+	if currentModule != "github.com/simnalamburt/module" {
+		t.Errorf("Unexpected module name")
+	}
+	if moduleDir != "C:\\Users\\work\\module" {
+		t.Errorf("Unexpected module path")
+	}
+	expected := []xcaddy.Replace{
+		xcaddy.NewReplace("replacetest1@v1.2.3", "golang.org/x/example@v0.0.0-20210811190340-787a929d5a0d"),
+		xcaddy.NewReplace("replacetest2@v0.0.1", "golang.org/x/example@v0.0.0-20210407023211-09c3a5e06b5d"),
+		xcaddy.NewReplace("replacetest3@v1.2.3", "C:\\Users\\work\\module\\fork1"),
+		xcaddy.NewReplace("github.com/simnalamburt/module", "C:\\Users\\work\\module"),
+		xcaddy.NewReplace("replacetest4@v0.0.1", "C:\\go\\fork2"),
+		xcaddy.NewReplace("replacetest5@v1.2.3", "C:\\Users\\work\\module\\fork3"),
+	}
+	if !reflect.DeepEqual(replacements, expected) {
+		t.Errorf("Expected replacements '%v' but got '%v'", expected, replacements)
+	}
+}