diff --git a/go.mod b/go.mod index 0bd5acbc3..42ddcf09e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( codeberg.org/gruf/go-debug v1.3.0 codeberg.org/gruf/go-errors/v2 v2.3.2 codeberg.org/gruf/go-fastcopy v1.1.3 - codeberg.org/gruf/go-ffmpreg v0.2.4 + codeberg.org/gruf/go-ffmpreg v0.2.5 codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf codeberg.org/gruf/go-kv v1.6.4 codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f @@ -56,7 +56,7 @@ require ( github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8 github.com/tdewolff/minify/v2 v2.20.37 github.com/technologize/otel-go-contrib v1.1.1 - github.com/tetratelabs/wazero v1.7.3 + github.com/tetratelabs/wazero v1.8.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 github.com/ulule/limiter/v3 v3.11.2 github.com/uptrace/bun v1.2.1 diff --git a/go.sum b/go.sum index 45069952a..364c54a02 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ codeberg.org/gruf/go-fastcopy v1.1.3 h1:Jo9VTQjI6KYimlw25PPc7YLA3Xm+XMQhaHwKnM7x codeberg.org/gruf/go-fastcopy v1.1.3/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s= codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q5Qo9c0= codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q= -codeberg.org/gruf/go-ffmpreg v0.2.4 h1:9NR0a5a0RjiIpyQgsqmHen6oadABADv04BWt7dr9kuE= -codeberg.org/gruf/go-ffmpreg v0.2.4/go.mod h1:oPMfBkOK7xmR/teT/dKW6SeMFpRos9ceR/OuUrxBfcQ= +codeberg.org/gruf/go-ffmpreg v0.2.5 h1:suQJ8VdWLkqUhDhHJEdOHMdLqvPisUxcAJiKrxjc6KQ= +codeberg.org/gruf/go-ffmpreg v0.2.5/go.mod h1:sViRI0BYK2B8PJw4BrOg7DquPD71mZjDfffRAFcDtvk= codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf h1:84s/ii8N6lYlskZjHH+DG6jyia8w2mXMZlRwFn8Gs3A= codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf/go.mod h1:zZAICsp5rY7+hxnws2V0ePrWxE0Z2Z/KXcN3p/RQCfk= codeberg.org/gruf/go-kv v1.6.4 h1:3NZiW8HVdBM3kpOiLb7XfRiihnzZWMAixdCznguhILk= @@ -552,8 +552,8 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/technologize/otel-go-contrib v1.1.1 h1:wZH9aSPNWZWIkEh3vfaKfMb15AJ80jJ1aVj/4GZdqIw= github.com/technologize/otel-go-contrib v1.1.1/go.mod h1:dCN/wj2WyUO8aFZFdIN+6tfJHImjTML/8r2YVYAy3So= -github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= -github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g= +github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo= diff --git a/vendor/codeberg.org/gruf/go-ffmpreg/wasm/instance.go b/vendor/codeberg.org/gruf/go-ffmpreg/wasm/instance.go index 7900ecd08..702fc856b 100644 --- a/vendor/codeberg.org/gruf/go-ffmpreg/wasm/instance.go +++ b/vendor/codeberg.org/gruf/go-ffmpreg/wasm/instance.go @@ -10,6 +10,10 @@ ) type Args struct { + // Optional further module configuration function. + // (e.g. to mount filesystem dir, set env vars, etc). + Config func(wazero.ModuleConfig) wazero.ModuleConfig + // Standard FDs. Stdin io.Reader Stdout io.Writer @@ -17,10 +21,6 @@ type Args struct { // CLI args. Args []string - - // Optional further module configuration function. - // (e.g. to mount filesystem dir, set env vars, etc). - Config func(wazero.ModuleConfig) wazero.ModuleConfig } type Instantiator struct { diff --git a/vendor/github.com/tetratelabs/wazero/Makefile b/vendor/github.com/tetratelabs/wazero/Makefile index e5ae8a261..17e264e01 100644 --- a/vendor/github.com/tetratelabs/wazero/Makefile +++ b/vendor/github.com/tetratelabs/wazero/Makefile @@ -1,7 +1,7 @@ -gofumpt := mvdan.cc/gofumpt@v0.5.0 +gofumpt := mvdan.cc/gofumpt@v0.6.0 gosimports := github.com/rinchsan/gosimports/cmd/gosimports@v0.3.8 -golangci_lint := github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 +golangci_lint := github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.0 asmfmt := github.com/klauspost/asmfmt/cmd/asmfmt@v1.3.2 # sync this with netlify.toml! hugo := github.com/gohugoio/hugo@v0.115.2 @@ -20,22 +20,6 @@ main_packages := $(sort $(foreach f,$(dir $(main_sources)),$(if $(findstring ./, go_test_options ?= -timeout 300s -ensureCompilerFastest := -ldflags '-X github.com/tetratelabs/wazero/internal/integration_test/vs.ensureCompilerFastest=true' -.PHONY: bench -bench: - @go build ./internal/integration_test/bench/... - @# Don't use -test.benchmem as it isn't accurate when comparing against CGO libs - @for d in vs/time vs/wasmedge vs/wasmtime ; do \ - cd ./internal/integration_test/$$d ; \ - go test -bench=. . -tags='wasmedge' $(ensureCompilerFastest) ; \ - cd - ;\ - done - -bench_testdata_dir := internal/integration_test/bench/testdata -.PHONY: build.bench -build.bench: - @tinygo build -o $(bench_testdata_dir)/case.wasm -scheduler=none --no-debug -target=wasi $(bench_testdata_dir)/case.go - .PHONY: test.examples test.examples: @go test $(go_test_options) ./examples/... ./imports/assemblyscript/example/... ./imports/emscripten/... ./imports/wasi_snapshot_preview1/example/... @@ -183,7 +167,7 @@ build.spectest.threads: .PHONY: test test: - @go test $(go_test_options) $$(go list ./... | grep -vE '$(spectest_v1_dir)|$(spectest_v2_dir)') + @go test $(go_test_options) ./... @cd internal/version/testdata && go test $(go_test_options) ./... @cd internal/integration_test/fuzz/wazerolib && CGO_ENABLED=0 WASM_BINARY_PATH=testdata/test.wasm go test ./... @@ -194,17 +178,6 @@ coverage: ## Generate test coverage @go test -coverprofile=coverage.txt -covermode=atomic --coverpkg=$(coverpkg) $(main_packages) @go tool cover -func coverage.txt -.PHONY: spectest -spectest: - @$(MAKE) spectest.v1 - @$(MAKE) spectest.v2 - -spectest.v1: - @go test $(go_test_options) $$(go list ./... | grep $(spectest_v1_dir)) - -spectest.v2: - @go test $(go_test_options) $$(go list ./... | grep $(spectest_v2_dir)) - golangci_lint_path := $(shell go env GOPATH)/bin/golangci-lint $(golangci_lint_path): @@ -214,7 +187,7 @@ golangci_lint_goarch ?= $(shell go env GOARCH) .PHONY: lint lint: $(golangci_lint_path) - @GOARCH=$(golangci_lint_goarch) CGO_ENABLED=0 $(golangci_lint_path) run --timeout 5m + @GOARCH=$(golangci_lint_goarch) CGO_ENABLED=0 $(golangci_lint_path) run --timeout 5m -E testableexamples .PHONY: format format: diff --git a/vendor/github.com/tetratelabs/wazero/README.md b/vendor/github.com/tetratelabs/wazero/README.md index 657da2959..f020be99a 100644 --- a/vendor/github.com/tetratelabs/wazero/README.md +++ b/vendor/github.com/tetratelabs/wazero/README.md @@ -1,6 +1,6 @@ # wazero: the zero dependency WebAssembly runtime for Go developers -[![WebAssembly Core Specification Test](https://github.com/tetratelabs/wazero/actions/workflows/spectest.yaml/badge.svg)](https://github.com/tetratelabs/wazero/actions/workflows/spectest.yaml) [![Go Reference](https://pkg.go.dev/badge/github.com/tetratelabs/wazero.svg)](https://pkg.go.dev/github.com/tetratelabs/wazero) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Go Reference](https://pkg.go.dev/badge/github.com/tetratelabs/wazero.svg)](https://pkg.go.dev/github.com/tetratelabs/wazero) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) WebAssembly is a way to safely run code compiled in other languages. Runtimes execute WebAssembly Modules (Wasm), which are most often binaries with a `.wasm` diff --git a/vendor/github.com/tetratelabs/wazero/api/wasm.go b/vendor/github.com/tetratelabs/wazero/api/wasm.go index c66b582fa..d99c1a756 100644 --- a/vendor/github.com/tetratelabs/wazero/api/wasm.go +++ b/vendor/github.com/tetratelabs/wazero/api/wasm.go @@ -151,9 +151,13 @@ type Module interface { // ExportedFunction returns a function exported from this module or nil if it wasn't. // - // Note: The default wazero.ModuleConfig attempts to invoke `_start`, which - // in rare cases can close the module. When in doubt, check IsClosed prior - // to invoking a function export after instantiation. + // # Notes + // - The default wazero.ModuleConfig attempts to invoke `_start`, which + // in rare cases can close the module. When in doubt, check IsClosed prior + // to invoking a function export after instantiation. + // - The semantics of host functions assumes the existence of an "importing module" because, for example, the host function needs access to + // the memory of the importing module. Therefore, direct use of ExportedFunction is forbidden for host modules. + // Practically speaking, it is usually meaningless to directly call a host function from Go code as it is already somewhere in Go code. ExportedFunction(name string) Function // ExportedFunctionDefinitions returns all the exported function diff --git a/vendor/github.com/tetratelabs/wazero/builder.go b/vendor/github.com/tetratelabs/wazero/builder.go index f64afabdf..b60a9e097 100644 --- a/vendor/github.com/tetratelabs/wazero/builder.go +++ b/vendor/github.com/tetratelabs/wazero/builder.go @@ -179,6 +179,9 @@ type HostFunctionBuilder interface { // are deferred until Compile. // - Functions are indexed in order of calls to NewFunctionBuilder as // insertion ordering is needed by ABI such as Emscripten (invoke_*). +// - The semantics of host functions assumes the existence of an "importing module" because, for example, the host function needs access to +// the memory of the importing module. Therefore, direct use of ExportedFunction is forbidden for host modules. +// Practically speaking, it is usually meaningless to directly call a host function from Go code as it is already somewhere in Go code. type HostModuleBuilder interface { // Note: until golang/go#5860, we can't use example tests to embed code in interface godocs. @@ -341,12 +344,24 @@ func (b *hostModuleBuilder) Compile(ctx context.Context) (CompiledModule, error) return c, nil } +// hostModuleInstance is a wrapper around api.Module that prevents calling ExportedFunction. +type hostModuleInstance struct{ api.Module } + +// ExportedFunction implements api.Module ExportedFunction. +func (h hostModuleInstance) ExportedFunction(name string) api.Function { + panic("calling ExportedFunction is forbidden on host modules. See the note on ExportedFunction interface") +} + // Instantiate implements HostModuleBuilder.Instantiate func (b *hostModuleBuilder) Instantiate(ctx context.Context) (api.Module, error) { if compiled, err := b.Compile(ctx); err != nil { return nil, err } else { compiled.(*compiledModule).closeWithModule = true - return b.r.InstantiateModule(ctx, compiled, NewModuleConfig()) + m, err := b.r.InstantiateModule(ctx, compiled, NewModuleConfig()) + if err != nil { + return nil, err + } + return hostModuleInstance{m}, nil } } diff --git a/vendor/github.com/tetratelabs/wazero/cache.go b/vendor/github.com/tetratelabs/wazero/cache.go index 2d1b4e3b9..83cdb94ef 100644 --- a/vendor/github.com/tetratelabs/wazero/cache.go +++ b/vendor/github.com/tetratelabs/wazero/cache.go @@ -24,6 +24,13 @@ // All implementations are in wazero. // - Instances of this can be reused across multiple runtimes, if configured // via RuntimeConfig. +// - The cache check happens before the compilation, so if multiple Goroutines are +// trying to compile the same module simultaneously, it is possible that they +// all compile the module. The design here is that the lock isn't held for the action "Compile" +// but only for checking and saving the compiled result. Therefore, we strongly recommend that the embedder +// does the centralized compilation in a single Goroutines (or multiple Goroutines per Wasm binary) to generate cache rather than +// trying to Compile in parallel for a single module. In other words, we always recommend to produce CompiledModule +// share it across multiple Goroutines to avoid trying to compile the same module simultaneously. type CompilationCache interface{ api.Closer } // NewCompilationCache returns a new CompilationCache to be passed to RuntimeConfig. diff --git a/vendor/github.com/tetratelabs/wazero/config.go b/vendor/github.com/tetratelabs/wazero/config.go index d3656849c..ea7b84f44 100644 --- a/vendor/github.com/tetratelabs/wazero/config.go +++ b/vendor/github.com/tetratelabs/wazero/config.go @@ -495,7 +495,20 @@ type ModuleConfig interface { WithFSConfig(FSConfig) ModuleConfig // WithName configures the module name. Defaults to what was decoded from - // the name section. Empty string ("") clears any name. + // the name section. Duplicate names are not allowed in a single Runtime. + // + // Calling this with the empty string "" makes the module anonymous. + // That is useful when you want to instantiate the same CompiledModule multiple times like below: + // + // for i := 0; i < N; i++ { + // // Instantiate a new Wasm module from the already compiled `compiledWasm` anonymously without a name. + // instance, err := r.InstantiateModule(ctx, compiledWasm, wazero.NewModuleConfig().WithName("")) + // // .... + // } + // + // See the `concurrent-instantiation` example for a complete usage. + // + // Non-empty named modules are available for other modules to import by name. WithName(string) ModuleConfig // WithStartFunctions configures the functions to call after the module is diff --git a/vendor/github.com/tetratelabs/wazero/experimental/importresolver.go b/vendor/github.com/tetratelabs/wazero/experimental/importresolver.go new file mode 100644 index 000000000..36c0e22b1 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/importresolver.go @@ -0,0 +1,19 @@ +package experimental + +import ( + "context" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/expctxkeys" +) + +// ImportResolver is an experimental func type that, if set, +// will be used as the first step in resolving imports. +// See issue 2294. +// If the import name is not found, it should return nil. +type ImportResolver func(name string) api.Module + +// WithImportResolver returns a new context with the given ImportResolver. +func WithImportResolver(ctx context.Context, resolver ImportResolver) context.Context { + return context.WithValue(ctx, expctxkeys.ImportResolverKey{}, resolver) +} diff --git a/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go b/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go index 1ec0d81b3..150f75cc1 100644 --- a/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go +++ b/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go @@ -1962,17 +1962,11 @@ func pathSymlinkFn(_ context.Context, mod api.Module, params []uint64) experimen return dir.FS.Symlink( // Do not join old path since it's only resolved when dereference the link created here. // And the dereference result depends on the opening directory's file descriptor at that point. - bufToStr(oldPathBuf), + unsafe.String(&oldPathBuf[0], int(oldPathLen)), newPathName, ) } -// bufToStr converts the given byte slice as string unsafely. -func bufToStr(buf []byte) string { - // TODO: use unsafe.String after flooring Go 1.20. - return *(*string)(unsafe.Pointer(&buf)) -} - // pathUnlinkFile is the WASI function named PathUnlinkFileName which unlinks a // file. // diff --git a/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go b/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go index d09f30245..4f96af2df 100644 --- a/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go +++ b/vendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go @@ -68,9 +68,7 @@ func pollOneoffFn(_ context.Context, mod api.Module, params []uint64) sys.Errno } outBuf, ok := mem.Read(out, nsubscriptions*32) // zero-out all buffer before writing - for i := range outBuf { - outBuf[i] = 0 - } + clear(outBuf) if !ok { return sys.EFAULT diff --git a/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go b/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go index 542958bc7..03761e6ec 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go +++ b/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go @@ -154,11 +154,6 @@ func (t *Table[Key, Item]) Range(f func(Key, Item) bool) { // Reset clears the content of the table. func (t *Table[Key, Item]) Reset() { - for i := range t.masks { - t.masks[i] = 0 - } - var zero Item - for i := range t.items { - t.items[i] = zero - } + clear(t.masks) + clear(t.items) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go index 56dfac620..4e20e4b2c 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go @@ -26,11 +26,14 @@ type ( controlFrame struct { frameID uint32 - // originalStackLen holds the number of values on the stack + // originalStackLenWithoutParam holds the number of values on the stack // when Start executing this control frame minus params for the block. originalStackLenWithoutParam int - blockType *wasm.FunctionType - kind controlFrameKind + // originalStackLenWithoutParamUint64 is almost the same as originalStackLenWithoutParam + // except that it holds the number of values on the stack in uint64. + originalStackLenWithoutParamUint64 int + blockType *wasm.FunctionType + kind controlFrameKind } controlFrames struct{ frames []controlFrame } ) @@ -157,9 +160,11 @@ type compiler struct { enabledFeatures api.CoreFeatures callFrameStackSizeInUint64 int stack []unsignedType - currentFrameID uint32 - controlFrames controlFrames - unreachableState struct { + // stackLenInUint64 is the length of the stack in uint64. + stackLenInUint64 int + currentFrameID uint32 + controlFrames controlFrames + unreachableState struct { on bool depth int } @@ -341,6 +346,7 @@ funcIndex := c.next c.pc = 0 c.currentOpPC = 0 c.currentFrameID = 0 + c.stackLenInUint64 = 0 c.unreachableState.on, c.unreachableState.depth = false, 0 if err := c.compile(sig, code.Body, code.LocalTypes, code.BodyOffsetInCodeSection); err != nil { @@ -449,10 +455,11 @@ func (c *compiler) handleInstruction() error { // Create a new frame -- entering this block. frame := controlFrame{ - frameID: c.nextFrameID(), - originalStackLenWithoutParam: len(c.stack) - len(bt.Params), - kind: controlFrameKindBlockWithoutContinuationLabel, - blockType: bt, + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + originalStackLenWithoutParamUint64: c.stackLenInUint64 - bt.ParamNumInUint64, + kind: controlFrameKindBlockWithoutContinuationLabel, + blockType: bt, } c.controlFrames.push(frame) @@ -473,10 +480,11 @@ func (c *compiler) handleInstruction() error { // Create a new frame -- entering loop. frame := controlFrame{ - frameID: c.nextFrameID(), - originalStackLenWithoutParam: len(c.stack) - len(bt.Params), - kind: controlFrameKindLoop, - blockType: bt, + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + originalStackLenWithoutParamUint64: c.stackLenInUint64 - bt.ParamNumInUint64, + kind: controlFrameKindLoop, + blockType: bt, } c.controlFrames.push(frame) @@ -515,8 +523,9 @@ func (c *compiler) handleInstruction() error { // Create a new frame -- entering if. frame := controlFrame{ - frameID: c.nextFrameID(), - originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + originalStackLenWithoutParamUint64: c.stackLenInUint64 - bt.ParamNumInUint64, // Note this will be set to controlFrameKindIfWithElse // when else opcode found later. kind: controlFrameKindIfWithoutElse, @@ -543,7 +552,7 @@ func (c *compiler) handleInstruction() error { // If it is currently in unreachable, and the non-nested if, // reset the stack so we can correctly handle the else block. top := c.controlFrames.top() - c.stack = c.stack[:top.originalStackLenWithoutParam] + c.stackSwitchAt(top) top.kind = controlFrameKindIfWithElse // Re-push the parameters to the if block so that else block can use them. @@ -572,7 +581,7 @@ func (c *compiler) handleInstruction() error { // Reset the stack manipulated by the then block, and re-push the block param types to the stack. - c.stack = c.stack[:frame.originalStackLenWithoutParam] + c.stackSwitchAt(frame) for _, t := range frame.blockType.Params { c.stackPush(wasmValueTypeTounsignedType(t)) } @@ -601,7 +610,7 @@ func (c *compiler) handleInstruction() error { return nil } - c.stack = c.stack[:frame.originalStackLenWithoutParam] + c.stackSwitchAt(frame) for _, t := range frame.blockType.Results { c.stackPush(wasmValueTypeTounsignedType(t)) } @@ -628,7 +637,7 @@ func (c *compiler) handleInstruction() error { // We need to reset the stack so that // the values pushed inside the block. dropOp := newOperationDrop(c.getFrameDropRange(frame, true)) - c.stack = c.stack[:frame.originalStackLenWithoutParam] + c.stackSwitchAt(frame) // Push the result types onto the stack. for _, t := range frame.blockType.Results { @@ -3505,6 +3514,11 @@ func (c *compiler) stackPeek() (ret unsignedType) { return } +func (c *compiler) stackSwitchAt(frame *controlFrame) { + c.stack = c.stack[:frame.originalStackLenWithoutParam] + c.stackLenInUint64 = frame.originalStackLenWithoutParamUint64 +} + func (c *compiler) stackPop() (ret unsignedType) { // No need to check stack bound // as we can assume that all the operations @@ -3512,11 +3526,13 @@ func (c *compiler) stackPop() (ret unsignedType) { // at module validation phase. ret = c.stack[len(c.stack)-1] c.stack = c.stack[:len(c.stack)-1] + c.stackLenInUint64 -= 1 + int(unsignedTypeV128&ret>>2) return } func (c *compiler) stackPush(ts unsignedType) { c.stack = append(c.stack, ts) + c.stackLenInUint64 += 1 + int(unsignedTypeV128&ts>>2) } // emit adds the operations into the result. @@ -3565,7 +3581,7 @@ func (c *compiler) emitDefaultValue(t wasm.ValueType) { // of the n-th local. func (c *compiler) localDepth(index wasm.Index) int { height := c.localIndexToStackHeightInUint64[index] - return c.stackLenInUint64(len(c.stack)) - 1 - int(height) + return c.stackLenInUint64 - 1 - height } func (c *compiler) localType(index wasm.Index) (t wasm.ValueType) { @@ -3592,14 +3608,7 @@ func (c *compiler) getFrameDropRange(frame *controlFrame, isEnd bool) inclusiveR } else { start = frame.blockType.ResultNumInUint64 } - var end int - if frame.kind == controlFrameKindFunction { - // On the function return, we eliminate all the contents on the stack - // including locals (existing below of frame.originalStackLen) - end = c.stackLenInUint64(len(c.stack)) - 1 - } else { - end = c.stackLenInUint64(len(c.stack)) - 1 - c.stackLenInUint64(frame.originalStackLenWithoutParam) - } + end := c.stackLenInUint64 - 1 - frame.originalStackLenWithoutParamUint64 if start <= end { return inclusiveRange{Start: int32(start), End: int32(end)} } else { @@ -3607,17 +3616,6 @@ func (c *compiler) getFrameDropRange(frame *controlFrame, isEnd bool) inclusiveR } } -func (c *compiler) stackLenInUint64(ceil int) (ret int) { - for i := 0; i < ceil; i++ { - if c.stack[i] == unsignedTypeV128 { - ret += 2 - } else { - ret++ - } - } - return -} - func (c *compiler) readMemoryArg(tag string) (memoryArg, error) { c.result.UsesMemory = true alignment, num, err := leb128.LoadUint32(c.body[c.pc+1:]) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go index 18c5f4252..ee0b453ca 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go @@ -3901,14 +3901,9 @@ func() { case operationKindV128Dot: x2Hi, x2Lo := ce.popValue(), ce.popValue() x1Hi, x1Lo := ce.popValue(), ce.popValue() - ce.pushValue( - uint64(uint32(int32(int16(x1Lo>>0))*int32(int16(x2Lo>>0))+int32(int16(x1Lo>>16))*int32(int16(x2Lo>>16)))) | - (uint64(uint32(int32(int16(x1Lo>>32))*int32(int16(x2Lo>>32))+int32(int16(x1Lo>>48))*int32(int16(x2Lo>>48)))) << 32), - ) - ce.pushValue( - uint64(uint32(int32(int16(x1Hi>>0))*int32(int16(x2Hi>>0))+int32(int16(x1Hi>>16))*int32(int16(x2Hi>>16)))) | - (uint64(uint32(int32(int16(x1Hi>>32))*int32(int16(x2Hi>>32))+int32(int16(x1Hi>>48))*int32(int16(x2Hi>>48)))) << 32), - ) + lo, hi := v128Dot(x1Hi, x1Lo, x2Hi, x2Lo) + ce.pushValue(lo) + ce.pushValue(hi) frame.pc++ case operationKindV128ITruncSatFromF: hi, lo := ce.popValue(), ce.popValue() @@ -4584,3 +4579,18 @@ func (ce *callEngine) callGoFuncWithStack(ctx context.Context, m *wasm.ModuleIns ce.stack = ce.stack[0 : len(ce.stack)-shrinkLen] } } + +// v128Dot performs a dot product of two 64-bit vectors. +// Note: for some reason (which I suspect is due to a bug in Go compiler's regalloc), +// inlining this function causes a bug which happens **only when** we run with -race AND arm64 AND Go 1.22. +func v128Dot(x1Hi, x1Lo, x2Hi, x2Lo uint64) (uint64, uint64) { + r1 := int32(int16(x1Lo>>0)) * int32(int16(x2Lo>>0)) + r2 := int32(int16(x1Lo>>16)) * int32(int16(x2Lo>>16)) + r3 := int32(int16(x1Lo>>32)) * int32(int16(x2Lo>>32)) + r4 := int32(int16(x1Lo>>48)) * int32(int16(x2Lo>>48)) + r5 := int32(int16(x1Hi>>0)) * int32(int16(x2Hi>>0)) + r6 := int32(int16(x1Hi>>16)) * int32(int16(x2Hi>>16)) + r7 := int32(int16(x1Hi>>32)) * int32(int16(x2Hi>>32)) + r8 := int32(int16(x1Hi>>48)) * int32(int16(x2Hi>>48)) + return uint64(uint32(r1+r2)) | (uint64(uint32(r3+r4)) << 32), uint64(uint32(r5+r6)) | (uint64(uint32(r7+r8)) << 32) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go index 59bbfe02d..62d365015 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go @@ -69,7 +69,7 @@ type Compiler interface { AllocateVReg(typ ssa.Type) regalloc.VReg // ValueDefinition returns the definition of the given value. - ValueDefinition(ssa.Value) *SSAValueDefinition + ValueDefinition(ssa.Value) SSAValueDefinition // VRegOf returns the virtual register of the given ssa.Value. VRegOf(value ssa.Value) regalloc.VReg @@ -79,13 +79,13 @@ type Compiler interface { // MatchInstr returns true if the given definition is from an instruction with the given opcode, the current group ID, // and a refcount of 1. That means, the instruction can be merged/swapped within the current instruction group. - MatchInstr(def *SSAValueDefinition, opcode ssa.Opcode) bool + MatchInstr(def SSAValueDefinition, opcode ssa.Opcode) bool // MatchInstrOneOf is the same as MatchInstr but for multiple opcodes. If it matches one of ssa.Opcode, // this returns the opcode. Otherwise, this returns ssa.OpcodeInvalid. // // Note: caller should be careful to avoid excessive allocation on opcodes slice. - MatchInstrOneOf(def *SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode + MatchInstrOneOf(def SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode // AddRelocationInfo appends the relocation information for the function reference at the current buffer offset. AddRelocationInfo(funcRef ssa.FuncRef) @@ -126,10 +126,7 @@ type compiler struct { nextVRegID regalloc.VRegID // ssaValueToVRegs maps ssa.ValueID to regalloc.VReg. ssaValueToVRegs [] /* VRegID to */ regalloc.VReg - // ssaValueDefinitions maps ssa.ValueID to its definition. - ssaValueDefinitions []SSAValueDefinition - // ssaValueRefCounts is a cached list obtained by ssa.Builder.ValueRefCounts(). - ssaValueRefCounts []int + ssaValuesInfo []ssa.ValueInfo // returnVRegs is the list of virtual registers that store the return values. returnVRegs []regalloc.VReg varEdges [][2]regalloc.VReg @@ -206,15 +203,10 @@ func (c *compiler) setCurrentGroupID(gid ssa.InstructionGroupID) { // assignVirtualRegisters assigns a virtual register to each ssa.ValueID Valid in the ssa.Builder. func (c *compiler) assignVirtualRegisters() { builder := c.ssaBuilder - refCounts := builder.ValueRefCounts() - c.ssaValueRefCounts = refCounts + c.ssaValuesInfo = builder.ValuesInfo() - need := len(refCounts) - if need >= len(c.ssaValueToVRegs) { - c.ssaValueToVRegs = append(c.ssaValueToVRegs, make([]regalloc.VReg, need+1)...) - } - if need >= len(c.ssaValueDefinitions) { - c.ssaValueDefinitions = append(c.ssaValueDefinitions, make([]SSAValueDefinition, need+1)...) + if diff := len(c.ssaValuesInfo) - len(c.ssaValueToVRegs); diff > 0 { + c.ssaValueToVRegs = append(c.ssaValueToVRegs, make([]regalloc.VReg, diff+1)...) } for blk := builder.BlockIteratorReversePostOrderBegin(); blk != nil; blk = builder.BlockIteratorReversePostOrderNext() { @@ -225,40 +217,26 @@ func (c *compiler) assignVirtualRegisters() { typ := p.Type() vreg := c.AllocateVReg(typ) c.ssaValueToVRegs[pid] = vreg - c.ssaValueDefinitions[pid] = SSAValueDefinition{BlockParamValue: p, BlkParamVReg: vreg} c.ssaTypeOfVRegID[vreg.ID()] = p.Type() } // Assigns each value to a virtual register produced by instructions. for cur := blk.Root(); cur != nil; cur = cur.Next() { r, rs := cur.Returns() - var N int if r.Valid() { id := r.ID() ssaTyp := r.Type() typ := r.Type() vReg := c.AllocateVReg(typ) c.ssaValueToVRegs[id] = vReg - c.ssaValueDefinitions[id] = SSAValueDefinition{ - Instr: cur, - N: 0, - RefCount: refCounts[id], - } c.ssaTypeOfVRegID[vReg.ID()] = ssaTyp - N++ } for _, r := range rs { id := r.ID() ssaTyp := r.Type() vReg := c.AllocateVReg(ssaTyp) c.ssaValueToVRegs[id] = vReg - c.ssaValueDefinitions[id] = SSAValueDefinition{ - Instr: cur, - N: N, - RefCount: refCounts[id], - } c.ssaTypeOfVRegID[vReg.ID()] = ssaTyp - N++ } } } @@ -299,8 +277,12 @@ func (c *compiler) Init() { } // ValueDefinition implements Compiler.ValueDefinition. -func (c *compiler) ValueDefinition(value ssa.Value) *SSAValueDefinition { - return &c.ssaValueDefinitions[value.ID()] +func (c *compiler) ValueDefinition(value ssa.Value) SSAValueDefinition { + return SSAValueDefinition{ + V: value, + Instr: c.ssaBuilder.InstructionOfValue(value), + RefCount: c.ssaValuesInfo[value.ID()].RefCount, + } } // VRegOf implements Compiler.VRegOf. @@ -319,7 +301,7 @@ func (c *compiler) TypeOf(v regalloc.VReg) ssa.Type { } // MatchInstr implements Compiler.MatchInstr. -func (c *compiler) MatchInstr(def *SSAValueDefinition, opcode ssa.Opcode) bool { +func (c *compiler) MatchInstr(def SSAValueDefinition, opcode ssa.Opcode) bool { instr := def.Instr return def.IsFromInstr() && instr.Opcode() == opcode && @@ -328,7 +310,7 @@ func (c *compiler) MatchInstr(def *SSAValueDefinition, opcode ssa.Opcode) bool { } // MatchInstrOneOf implements Compiler.MatchInstrOneOf. -func (c *compiler) MatchInstrOneOf(def *SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode { +func (c *compiler) MatchInstrOneOf(def SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode { instr := def.Instr if !def.IsFromInstr() { return ssa.OpcodeInvalid diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go index 80e65668a..735cfa3d3 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go @@ -9,7 +9,7 @@ func (c *compiler) Lower() { c.assignVirtualRegisters() c.mach.SetCurrentABI(c.GetFunctionABI(c.ssaBuilder.Signature())) - c.mach.ExecutableContext().StartLoweringFunction(c.ssaBuilder.BlockIDMax()) + c.mach.StartLoweringFunction(c.ssaBuilder.BlockIDMax()) c.lowerBlocks() } @@ -20,12 +20,11 @@ func (c *compiler) lowerBlocks() { c.lowerBlock(blk) } - ectx := c.mach.ExecutableContext() // After lowering all blocks, we need to link adjacent blocks to layout one single instruction list. var prev ssa.BasicBlock for next := builder.BlockIteratorReversePostOrderBegin(); next != nil; next = builder.BlockIteratorReversePostOrderNext() { if prev != nil { - ectx.LinkAdjacentBlocks(prev, next) + c.mach.LinkAdjacentBlocks(prev, next) } prev = next } @@ -33,8 +32,7 @@ func (c *compiler) lowerBlocks() { func (c *compiler) lowerBlock(blk ssa.BasicBlock) { mach := c.mach - ectx := mach.ExecutableContext() - ectx.StartBlock(blk) + mach.StartBlock(blk) // We traverse the instructions in reverse order because we might want to lower multiple // instructions together. @@ -76,7 +74,7 @@ func (c *compiler) lowerBlock(blk ssa.BasicBlock) { default: mach.LowerInstr(cur) } - ectx.FlushPendingInstructions() + mach.FlushPendingInstructions() } // Finally, if this is the entry block, we have to insert copies of arguments from the real location to the VReg. @@ -84,7 +82,7 @@ func (c *compiler) lowerBlock(blk ssa.BasicBlock) { c.lowerFunctionArguments(blk) } - ectx.EndBlock() + mach.EndBlock() } // lowerBranches is called right after StartBlock and before any LowerInstr call if @@ -93,23 +91,24 @@ func (c *compiler) lowerBlock(blk ssa.BasicBlock) { // // See ssa.Instruction IsBranching, and the comment on ssa.BasicBlock. func (c *compiler) lowerBranches(br0, br1 *ssa.Instruction) { - ectx := c.mach.ExecutableContext() + mach := c.mach c.setCurrentGroupID(br0.GroupID()) c.mach.LowerSingleBranch(br0) - ectx.FlushPendingInstructions() + mach.FlushPendingInstructions() if br1 != nil { c.setCurrentGroupID(br1.GroupID()) c.mach.LowerConditionalBranch(br1) - ectx.FlushPendingInstructions() + mach.FlushPendingInstructions() } if br0.Opcode() == ssa.OpcodeJump { - _, args, target := br0.BranchData() + _, args, targetBlockID := br0.BranchData() argExists := len(args) != 0 if argExists && br1 != nil { panic("BUG: critical edge split failed") } + target := c.ssaBuilder.BasicBlock(targetBlockID) if argExists && target.ReturnBlock() { if len(args) > 0 { c.mach.LowerReturns(args) @@ -118,24 +117,25 @@ func (c *compiler) lowerBranches(br0, br1 *ssa.Instruction) { c.lowerBlockArguments(args, target) } } - ectx.FlushPendingInstructions() + mach.FlushPendingInstructions() } func (c *compiler) lowerFunctionArguments(entry ssa.BasicBlock) { - ectx := c.mach.ExecutableContext() + mach := c.mach c.tmpVals = c.tmpVals[:0] + data := c.ssaBuilder.ValuesInfo() for i := 0; i < entry.Params(); i++ { p := entry.Param(i) - if c.ssaValueRefCounts[p.ID()] > 0 { + if data[p.ID()].RefCount > 0 { c.tmpVals = append(c.tmpVals, p) } else { // If the argument is not used, we can just pass an invalid value. c.tmpVals = append(c.tmpVals, ssa.ValueInvalid) } } - c.mach.LowerParams(c.tmpVals) - ectx.FlushPendingInstructions() + mach.LowerParams(c.tmpVals) + mach.FlushPendingInstructions() } // lowerBlockArguments lowers how to pass arguments to the given successor block. @@ -152,12 +152,12 @@ func (c *compiler) lowerBlockArguments(args []ssa.Value, succ ssa.BasicBlock) { src := args[i] dstReg := c.VRegOf(dst) - srcDef := c.ssaValueDefinitions[src.ID()] - if srcDef.IsFromInstr() && srcDef.Instr.Constant() { + srcInstr := c.ssaBuilder.InstructionOfValue(src) + if srcInstr != nil && srcInstr.Constant() { c.constEdges = append(c.constEdges, struct { cInst *ssa.Instruction dst regalloc.VReg - }{cInst: srcDef.Instr, dst: dstReg}) + }{cInst: srcInstr, dst: dstReg}) } else { srcReg := c.VRegOf(src) // Even when the src=dst, insert the move so that we can keep such registers keep-alive. diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go deleted file mode 100644 index 8e9571b20..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go +++ /dev/null @@ -1,221 +0,0 @@ -package backend - -import ( - "fmt" - "math" - - "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" - "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" -) - -type ExecutableContext interface { - // StartLoweringFunction is called when the lowering of the given function is started. - // maximumBlockID is the maximum value of ssa.BasicBlockID existing in the function. - StartLoweringFunction(maximumBlockID ssa.BasicBlockID) - - // LinkAdjacentBlocks is called after finished lowering all blocks in order to create one single instruction list. - LinkAdjacentBlocks(prev, next ssa.BasicBlock) - - // StartBlock is called when the compilation of the given block is started. - // The order of this being called is the reverse post order of the ssa.BasicBlock(s) as we iterate with - // ssa.Builder BlockIteratorReversePostOrderBegin and BlockIteratorReversePostOrderEnd. - StartBlock(ssa.BasicBlock) - - // EndBlock is called when the compilation of the current block is finished. - EndBlock() - - // FlushPendingInstructions flushes the pending instructions to the buffer. - // This will be called after the lowering of each SSA Instruction. - FlushPendingInstructions() -} - -type ExecutableContextT[Instr any] struct { - CurrentSSABlk ssa.BasicBlock - - // InstrPool is the InstructionPool of instructions. - InstructionPool wazevoapi.Pool[Instr] - asNop func(*Instr) - setNext func(*Instr, *Instr) - setPrev func(*Instr, *Instr) - - // RootInstr is the root instruction of the executable. - RootInstr *Instr - labelPositionPool wazevoapi.Pool[LabelPosition[Instr]] - NextLabel Label - // LabelPositions maps a label to the instructions of the region which the label represents. - LabelPositions []*LabelPosition[Instr] - OrderedBlockLabels []*LabelPosition[Instr] - - // PerBlockHead and PerBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock. - PerBlockHead, PerBlockEnd *Instr - // PendingInstructions are the instructions which are not yet emitted into the instruction list. - PendingInstructions []*Instr - - // SsaBlockIDToLabels maps an SSA block ID to the label. - SsaBlockIDToLabels []Label -} - -func NewExecutableContextT[Instr any]( - resetInstruction func(*Instr), - setNext func(*Instr, *Instr), - setPrev func(*Instr, *Instr), - asNop func(*Instr), -) *ExecutableContextT[Instr] { - return &ExecutableContextT[Instr]{ - InstructionPool: wazevoapi.NewPool[Instr](resetInstruction), - asNop: asNop, - setNext: setNext, - setPrev: setPrev, - labelPositionPool: wazevoapi.NewPool[LabelPosition[Instr]](resetLabelPosition[Instr]), - NextLabel: LabelInvalid, - } -} - -func resetLabelPosition[T any](l *LabelPosition[T]) { - *l = LabelPosition[T]{} -} - -// StartLoweringFunction implements ExecutableContext. -func (e *ExecutableContextT[Instr]) StartLoweringFunction(max ssa.BasicBlockID) { - imax := int(max) - if len(e.SsaBlockIDToLabels) <= imax { - // Eagerly allocate labels for the blocks since the underlying slice will be used for the next iteration. - e.SsaBlockIDToLabels = append(e.SsaBlockIDToLabels, make([]Label, imax+1)...) - } -} - -func (e *ExecutableContextT[Instr]) StartBlock(blk ssa.BasicBlock) { - e.CurrentSSABlk = blk - - l := e.SsaBlockIDToLabels[e.CurrentSSABlk.ID()] - if l == LabelInvalid { - l = e.AllocateLabel() - e.SsaBlockIDToLabels[blk.ID()] = l - } - - end := e.allocateNop0() - e.PerBlockHead, e.PerBlockEnd = end, end - - labelPos := e.GetOrAllocateLabelPosition(l) - e.OrderedBlockLabels = append(e.OrderedBlockLabels, labelPos) - labelPos.Begin, labelPos.End = end, end - labelPos.SB = blk -} - -// EndBlock implements ExecutableContext. -func (e *ExecutableContextT[T]) EndBlock() { - // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions. - e.insertAtPerBlockHead(e.allocateNop0()) - - l := e.SsaBlockIDToLabels[e.CurrentSSABlk.ID()] - e.LabelPositions[l].Begin = e.PerBlockHead - - if e.CurrentSSABlk.EntryBlock() { - e.RootInstr = e.PerBlockHead - } -} - -func (e *ExecutableContextT[T]) insertAtPerBlockHead(i *T) { - if e.PerBlockHead == nil { - e.PerBlockHead = i - e.PerBlockEnd = i - return - } - e.setNext(i, e.PerBlockHead) - e.setPrev(e.PerBlockHead, i) - e.PerBlockHead = i -} - -// FlushPendingInstructions implements ExecutableContext. -func (e *ExecutableContextT[T]) FlushPendingInstructions() { - l := len(e.PendingInstructions) - if l == 0 { - return - } - for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order. - e.insertAtPerBlockHead(e.PendingInstructions[i]) - } - e.PendingInstructions = e.PendingInstructions[:0] -} - -func (e *ExecutableContextT[T]) Reset() { - e.labelPositionPool.Reset() - e.InstructionPool.Reset() - for i := range e.LabelPositions { - e.LabelPositions[i] = nil - } - e.PendingInstructions = e.PendingInstructions[:0] - e.OrderedBlockLabels = e.OrderedBlockLabels[:0] - e.RootInstr = nil - e.SsaBlockIDToLabels = e.SsaBlockIDToLabels[:0] - e.PerBlockHead, e.PerBlockEnd = nil, nil - e.NextLabel = LabelInvalid -} - -// AllocateLabel allocates an unused label. -func (e *ExecutableContextT[T]) AllocateLabel() Label { - e.NextLabel++ - return e.NextLabel -} - -func (e *ExecutableContextT[T]) GetOrAllocateLabelPosition(l Label) *LabelPosition[T] { - if len(e.LabelPositions) <= int(l) { - e.LabelPositions = append(e.LabelPositions, make([]*LabelPosition[T], int(l)+1-len(e.LabelPositions))...) - } - ret := e.LabelPositions[l] - if ret == nil { - ret = e.labelPositionPool.Allocate() - ret.L = l - e.LabelPositions[l] = ret - } - return ret -} - -func (e *ExecutableContextT[T]) GetOrAllocateSSABlockLabel(blk ssa.BasicBlock) Label { - if blk.ReturnBlock() { - return LabelReturn - } - l := e.SsaBlockIDToLabels[blk.ID()] - if l == LabelInvalid { - l = e.AllocateLabel() - e.SsaBlockIDToLabels[blk.ID()] = l - } - return l -} - -func (e *ExecutableContextT[T]) allocateNop0() *T { - i := e.InstructionPool.Allocate() - e.asNop(i) - return i -} - -// LinkAdjacentBlocks implements backend.Machine. -func (e *ExecutableContextT[T]) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { - prevLabelPos := e.LabelPositions[e.GetOrAllocateSSABlockLabel(prev)] - nextLabelPos := e.LabelPositions[e.GetOrAllocateSSABlockLabel(next)] - e.setNext(prevLabelPos.End, nextLabelPos.Begin) -} - -// LabelPosition represents the regions of the generated code which the label represents. -type LabelPosition[Instr any] struct { - SB ssa.BasicBlock - L Label - Begin, End *Instr - BinaryOffset int64 -} - -// Label represents a position in the generated code which is either -// a real instruction or the constant InstructionPool (e.g. jump tables). -// -// This is exactly the same as the traditional "label" in assembly code. -type Label uint32 - -const ( - LabelInvalid Label = 0 - LabelReturn Label = math.MaxUint32 -) - -// String implements backend.Machine. -func (l Label) String() string { - return fmt.Sprintf("L%d", l) -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go index 751050aff..96f035e58 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go @@ -14,7 +14,6 @@ // CompileGoFunctionTrampoline implements backend.Machine. func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte { - ectx := m.ectx argBegin := 1 // Skips exec context by default. if needModuleContextPtr { argBegin++ @@ -25,7 +24,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * m.currentABI = abi cur := m.allocateNop() - ectx.RootInstr = cur + m.rootInstr = cur // Execution context is always the first argument. execCtrPtr := raxVReg @@ -272,7 +271,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * cur = m.revertRBPRSP(cur) linkInstr(cur, m.allocateInstr().asRet()) - m.encodeWithoutSSA(ectx.RootInstr) + m.encodeWithoutSSA(m.rootInstr) return m.c.Buf() } @@ -347,10 +346,8 @@ func (m *machine) storeReturnAddressAndExit(cur *instruction, execCtx regalloc.V // CompileStackGrowCallSequence implements backend.Machine. func (m *machine) CompileStackGrowCallSequence() []byte { - ectx := m.ectx - cur := m.allocateNop() - ectx.RootInstr = cur + m.rootInstr = cur cur = m.setupRBPRSP(cur) @@ -379,7 +376,7 @@ func (m *machine) CompileStackGrowCallSequence() []byte { cur = m.revertRBPRSP(cur) linkInstr(cur, m.allocateInstr().asRet()) - m.encodeWithoutSSA(ectx.RootInstr) + m.encodeWithoutSSA(m.rootInstr) return m.c.Buf() } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go index d27e79c0e..6a3e58f51 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go @@ -17,16 +17,6 @@ type instruction struct { kind instructionKind } -// Next implements regalloc.Instr. -func (i *instruction) Next() regalloc.Instr { - return i.next -} - -// Prev implements regalloc.Instr. -func (i *instruction) Prev() regalloc.Instr { - return i.prev -} - // IsCall implements regalloc.Instr. func (i *instruction) IsCall() bool { return i.kind == call } @@ -36,9 +26,6 @@ func (i *instruction) IsIndirectCall() bool { return i.kind == callIndirect } // IsReturn implements regalloc.Instr. func (i *instruction) IsReturn() bool { return i.kind == ret } -// AddedBeforeRegAlloc implements regalloc.Instr. -func (i *instruction) AddedBeforeRegAlloc() bool { return i.addedBeforeRegAlloc } - // String implements regalloc.Instr. func (i *instruction) String() string { switch i.kind { @@ -651,26 +638,14 @@ func resetInstruction(i *instruction) { *i = instruction{} } -func setNext(i *instruction, next *instruction) { - i.next = next -} - -func setPrev(i *instruction, prev *instruction) { - i.prev = prev -} - -func asNop(i *instruction) { - i.kind = nop0 -} - -func (i *instruction) asNop0WithLabel(label backend.Label) *instruction { //nolint +func (i *instruction) asNop0WithLabel(label label) *instruction { //nolint i.kind = nop0 i.u1 = uint64(label) return i } -func (i *instruction) nop0Label() backend.Label { - return backend.Label(i.u1) +func (i *instruction) nop0Label() label { + return label(i.u1) } type instructionKind byte @@ -1161,7 +1136,7 @@ func (i *instruction) asJmp(target operand) *instruction { return i } -func (i *instruction) jmpLabel() backend.Label { +func (i *instruction) jmpLabel() label { switch i.kind { case jmp, jmpIf, lea, xmmUnaryRmR: return i.op1.label() diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go index bee673d25..befe8c643 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go @@ -130,9 +130,9 @@ func (m *machine) lowerAddendsToAmode(x, y addend, offBase uint32) *amode { } } -func (m *machine) lowerAddend(x *backend.SSAValueDefinition) addend { - if x.IsFromBlockParam() { - return addend{x.BlkParamVReg, 0, 0} +func (m *machine) lowerAddend(x backend.SSAValueDefinition) addend { + if !x.IsFromInstr() { + return addend{m.c.VRegOf(x.V), 0, 0} } // Ensure the addend is not referenced in multiple places; we will discard nested Iadds. op := m.c.MatchInstrOneOf(x, addendsMatchOpcodes[:]) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go index 61ae6f406..aeeb6b645 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go @@ -16,18 +16,13 @@ // NewBackend returns a new backend for arm64. func NewBackend() backend.Machine { - ectx := backend.NewExecutableContextT[instruction]( - resetInstruction, - setNext, - setPrev, - asNop, - ) - return &machine{ - ectx: ectx, + m := &machine{ cpuFeatures: platform.CpuFeatures, - regAlloc: regalloc.NewAllocator(regInfo), + regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo), spillSlots: map[regalloc.VRegID]int64{}, amodePool: wazevoapi.NewPool[amode](nil), + labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition), + instrPool: wazevoapi.NewPool[instruction](resetInstruction), constSwizzleMaskConstIndex: -1, constSqmulRoundSatIndex: -1, constI8x16SHLMaskTableIndex: -1, @@ -41,23 +36,46 @@ func NewBackend() backend.Machine { constExtAddPairwiseI16x8uMask1Index: -1, constExtAddPairwiseI16x8uMask2Index: -1, } + m.regAllocFn.m = m + return m } type ( // machine implements backend.Machine for amd64. machine struct { c backend.Compiler - ectx *backend.ExecutableContextT[instruction] stackBoundsCheckDisabled bool + instrPool wazevoapi.Pool[instruction] amodePool wazevoapi.Pool[amode] cpuFeatures platform.CpuFeatureFlags - regAlloc regalloc.Allocator - regAllocFn *backend.RegAllocFunction[*instruction, *machine] + regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn] + regAllocFn regAllocFn regAllocStarted bool + // labelPositionPool is the pool of labelPosition. The id is the label where + // if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID. + labelPositionPool wazevoapi.IDedPool[labelPosition] + // nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID + // so that we can have an identical label for the SSA block ID, which is useful for debugging. + nextLabel label + // rootInstr is the first instruction of the function. + rootInstr *instruction + // currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition. + currentLabelPos *labelPosition + // orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock. + orderedSSABlockLabelPos []*labelPosition + // returnLabelPos is the labelPosition for the return block. + returnLabelPos labelPosition + // perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock. + perBlockHead, perBlockEnd *instruction + // pendingInstructions are the instructions which are not yet emitted into the instruction list. + pendingInstructions []*instruction + // maxSSABlockID is the maximum ssa.BasicBlockID in the current function. + maxSSABlockID label + spillSlotSize int64 spillSlots map[regalloc.VRegID]int64 currentABI *backend.FunctionABI @@ -67,8 +85,11 @@ func NewBackend() backend.Machine { labelResolutionPends []labelResolutionPend + // jmpTableTargets holds the labels of the jump table targets. jmpTableTargets [][]uint32 - consts []_const + // jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table. + jmpTableTargetsNext int + consts []_const constSwizzleMaskConstIndex, constSqmulRoundSatIndex, constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex, @@ -79,9 +100,10 @@ func NewBackend() backend.Machine { } _const struct { - lo, hi uint64 - _var []byte - label *labelPosition + lo, hi uint64 + _var []byte + label label + labelPos *labelPosition } labelResolutionPend struct { @@ -90,22 +112,73 @@ func NewBackend() backend.Machine { // imm32Offset is the offset of the last 4 bytes of the instruction. imm32Offset int64 } - - labelPosition = backend.LabelPosition[instruction] ) -func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) backend.Label { +type ( + // label represents a position in the generated code which is either + // a real instruction or the constant InstructionPool (e.g. jump tables). + // + // This is exactly the same as the traditional "label" in assembly code. + label uint32 + + // labelPosition represents the regions of the generated code which the label represents. + // This implements regalloc.Block. + labelPosition struct { + // sb is not nil if this corresponds to a ssa.BasicBlock. + sb ssa.BasicBlock + // cur is used to walk through the instructions in the block during the register allocation. + cur, + // begin and end are the first and last instructions of the block. + begin, end *instruction + // binaryOffset is the offset in the binary where the label is located. + binaryOffset int64 + } +) + +// String implements backend.Machine. +func (l label) String() string { + return fmt.Sprintf("L%d", l) +} + +func resetLabelPosition(l *labelPosition) { + *l = labelPosition{} +} + +const labelReturn = math.MaxUint32 + +func ssaBlockLabel(sb ssa.BasicBlock) label { + if sb.ReturnBlock() { + return labelReturn + } + return label(sb.ID()) +} + +// getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block. +func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition { + if sb.ReturnBlock() { + m.returnLabelPos.sb = sb + return &m.returnLabelPos + } + + l := ssaBlockLabel(sb) + pos := m.labelPositionPool.GetOrAllocate(int(l)) + pos.sb = sb + return pos +} + +func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) label { index := *i if index == -1 { - label := m.allocateLabel() + l, pos := m.allocateLabel() index = len(m.consts) m.consts = append(m.consts, _const{ - _var: _var, - label: label, + _var: _var, + label: l, + labelPos: pos, }) *i = index } - return m.consts[index].label.L + return m.consts[index].label } // Reset implements backend.Machine. @@ -120,18 +193,20 @@ func (m *machine) Reset() { } m.stackBoundsCheckDisabled = false - m.ectx.Reset() - - m.regAllocFn.Reset() m.regAlloc.Reset() + m.labelPositionPool.Reset() + m.instrPool.Reset() m.regAllocStarted = false m.clobberedRegs = m.clobberedRegs[:0] m.spillSlotSize = 0 m.maxRequiredStackSizeForCalls = 0 + m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil + m.pendingInstructions = m.pendingInstructions[:0] + m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0] m.amodePool.Reset() - m.jmpTableTargets = m.jmpTableTargets[:0] + m.jmpTableTargetsNext = 0 m.constSwizzleMaskConstIndex = -1 m.constSqmulRoundSatIndex = -1 m.constI8x16SHLMaskTableIndex = -1 @@ -146,8 +221,63 @@ func (m *machine) Reset() { m.constExtAddPairwiseI16x8uMask2Index = -1 } -// ExecutableContext implements backend.Machine. -func (m *machine) ExecutableContext() backend.ExecutableContext { return m.ectx } +// StartLoweringFunction implements backend.Machine StartLoweringFunction. +func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) { + m.maxSSABlockID = label(maxBlockID) + m.nextLabel = label(maxBlockID) + 1 +} + +// LinkAdjacentBlocks implements backend.Machine. +func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { + prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next) + prevPos.end.next = nextPos.begin +} + +// StartBlock implements backend.Machine. +func (m *machine) StartBlock(blk ssa.BasicBlock) { + m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk) + labelPos := m.currentLabelPos + end := m.allocateNop() + m.perBlockHead, m.perBlockEnd = end, end + labelPos.begin, labelPos.end = end, end + m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos) +} + +// EndBlock implements ExecutableContext. +func (m *machine) EndBlock() { + // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions. + m.insertAtPerBlockHead(m.allocateNop()) + + m.currentLabelPos.begin = m.perBlockHead + + if m.currentLabelPos.sb.EntryBlock() { + m.rootInstr = m.perBlockHead + } +} + +func (m *machine) insertAtPerBlockHead(i *instruction) { + if m.perBlockHead == nil { + m.perBlockHead = i + m.perBlockEnd = i + return + } + + i.next = m.perBlockHead + m.perBlockHead.prev = i + m.perBlockHead = i +} + +// FlushPendingInstructions implements backend.Machine. +func (m *machine) FlushPendingInstructions() { + l := len(m.pendingInstructions) + if l == 0 { + return + } + for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order. + m.insertAtPerBlockHead(m.pendingInstructions[i]) + } + m.pendingInstructions = m.pendingInstructions[:0] +} // DisableStackCheck implements backend.Machine. func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } @@ -155,23 +285,17 @@ func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } // SetCompiler implements backend.Machine. func (m *machine) SetCompiler(c backend.Compiler) { m.c = c - m.regAllocFn = backend.NewRegAllocFunction[*instruction, *machine](m, c.SSABuilder(), c) + m.regAllocFn.ssaB = c.SSABuilder() } // SetCurrentABI implements backend.Machine. -func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { - m.currentABI = abi -} +func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { m.currentABI = abi } // RegAlloc implements backend.Machine. func (m *machine) RegAlloc() { rf := m.regAllocFn - for _, pos := range m.ectx.OrderedBlockLabels { - rf.AddBlock(pos.SB, pos.L, pos.Begin, pos.End) - } - m.regAllocStarted = true - m.regAlloc.DoAllocation(rf) + m.regAlloc.DoAllocation(&rf) // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes. m.spillSlotSize = (m.spillSlotSize + 15) &^ 15 } @@ -184,49 +308,54 @@ func (m *machine) InsertReturn() { // LowerSingleBranch implements backend.Machine. func (m *machine) LowerSingleBranch(b *ssa.Instruction) { - ectx := m.ectx switch b.Opcode() { case ssa.OpcodeJump: - _, _, targetBlk := b.BranchData() + _, _, targetBlkID := b.BranchData() if b.IsFallthroughJump() { return } jmp := m.allocateInstr() - target := ectx.GetOrAllocateSSABlockLabel(targetBlk) - if target == backend.LabelReturn { + target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID)) + if target == labelReturn { jmp.asRet() } else { jmp.asJmp(newOperandLabel(target)) } m.insert(jmp) case ssa.OpcodeBrTable: - index, target := b.BrTableData() - m.lowerBrTable(index, target) + index, targetBlkIDs := b.BrTableData() + m.lowerBrTable(index, targetBlkIDs) default: panic("BUG: unexpected branch opcode" + b.Opcode().String()) } } -func (m *machine) addJmpTableTarget(targets []ssa.BasicBlock) (index int) { - // TODO: reuse the slice! - labels := make([]uint32, len(targets)) - for j, target := range targets { - labels[j] = uint32(m.ectx.GetOrAllocateSSABlockLabel(target)) +func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) { + if m.jmpTableTargetsNext == len(m.jmpTableTargets) { + m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View()))) + } + + index = m.jmpTableTargetsNext + m.jmpTableTargetsNext++ + m.jmpTableTargets[index] = m.jmpTableTargets[index][:0] + for _, targetBlockID := range targets.View() { + target := m.c.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID)) + m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(ssaBlockLabel(target))) } - index = len(m.jmpTableTargets) - m.jmpTableTargets = append(m.jmpTableTargets, labels) return } var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp} -func (m *machine) lowerBrTable(index ssa.Value, targets []ssa.BasicBlock) { +func (m *machine) lowerBrTable(index ssa.Value, targets ssa.Values) { _v := m.getOperand_Reg(m.c.ValueDefinition(index)) v := m.copyToTmp(_v.reg()) + targetCount := len(targets.View()) + // First, we need to do the bounds check. maxIndex := m.c.AllocateVReg(ssa.TypeI32) - m.lowerIconst(maxIndex, uint64(len(targets)-1), false) + m.lowerIconst(maxIndex, uint64(targetCount-1), false) cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false) m.insert(cmp) @@ -255,23 +384,22 @@ func (m *machine) lowerBrTable(index ssa.Value, targets []ssa.BasicBlock) { jmpTable := m.allocateInstr() targetSliceIndex := m.addJmpTableTarget(targets) - jmpTable.asJmpTableSequence(targetSliceIndex, len(targets)) + jmpTable.asJmpTableSequence(targetSliceIndex, targetCount) m.insert(jmpTable) } // LowerConditionalBranch implements backend.Machine. func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { - exctx := m.ectx - cval, args, targetBlk := b.BranchData() + cval, args, targetBlkID := b.BranchData() if len(args) > 0 { panic(fmt.Sprintf( "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", - exctx.CurrentSSABlk, - targetBlk, + m.currentLabelPos.sb, + targetBlkID, )) } - target := exctx.GetOrAllocateSSABlockLabel(targetBlk) + target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID)) cvalDef := m.c.ValueDefinition(cval) switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) { @@ -1272,9 +1400,9 @@ func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) { } load := m.allocateInstr() - constLabel := m.allocateLabel() - m.consts = append(m.consts, _const{label: constLabel, lo: lo, hi: hi}) - load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(constLabel.L)), dst) + l, pos := m.allocateLabel() + m.consts = append(m.consts, _const{label: l, labelPos: pos, lo: lo, hi: hi}) + load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(l)), dst) m.insert(load) } @@ -1473,21 +1601,24 @@ func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l)) } -func (m *machine) tryLowerBandToFlag(x, y *backend.SSAValueDefinition) (ok bool) { - var target *backend.SSAValueDefinition +func (m *machine) tryLowerBandToFlag(x, y backend.SSAValueDefinition) (ok bool) { + var target backend.SSAValueDefinition + var got bool if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 { if m.c.MatchInstr(y, ssa.OpcodeBand) { target = y + got = true } } if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 { if m.c.MatchInstr(x, ssa.OpcodeBand) { target = x + got = true } } - if target == nil { + if !got { return false } @@ -1522,7 +1653,7 @@ func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) ( return } -func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel backend.Label) { +func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel label) { exitCodeReg := rbpVReg saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg) @@ -1819,9 +1950,9 @@ func (m *machine) lowerCall(si *ssa.Instruction) { // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the // caller side of the function call. -func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, stackSlotSize int64) { +func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, stackSlotSize int64) { arg := &a.Args[argIndex] - if def != nil && def.IsFromInstr() { + if def.IsFromInstr() { // Constant instructions are inlined. if inst := def.Instr; inst.Constant() { m.insertLoadConstant(inst, reg) @@ -1904,25 +2035,20 @@ func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) { // Format implements backend.Machine. func (m *machine) Format() string { - ectx := m.ectx - begins := map[*instruction]backend.Label{} - for _, pos := range ectx.LabelPositions { + begins := map[*instruction]label{} + for l := label(0); l < m.nextLabel; l++ { + pos := m.labelPositionPool.Get(int(l)) if pos != nil { - begins[pos.Begin] = pos.L + begins[pos.begin] = l } } - irBlocks := map[backend.Label]ssa.BasicBlockID{} - for i, l := range ectx.SsaBlockIDToLabels { - irBlocks[l] = ssa.BasicBlockID(i) - } - var lines []string - for cur := ectx.RootInstr; cur != nil; cur = cur.next { + for cur := m.rootInstr; cur != nil; cur = cur.next { if l, ok := begins[cur]; ok { var labelStr string - if blkID, ok := irBlocks[l]; ok { - labelStr = fmt.Sprintf("%s (SSA Block: %s):", l, blkID) + if l <= m.maxSSABlockID { + labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, l) } else { labelStr = fmt.Sprintf("%s:", l) } @@ -1935,9 +2061,9 @@ func (m *machine) Format() string { } for _, vc := range m.consts { if vc._var == nil { - lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label.L, vc.lo, vc.hi)) + lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label, vc.lo, vc.hi)) } else { - lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label.L, vc._var)) + lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label, vc._var)) } } return "\n" + strings.Join(lines, "\n") + "\n" @@ -1945,18 +2071,14 @@ func (m *machine) Format() string { func (m *machine) encodeWithoutSSA(root *instruction) { m.labelResolutionPends = m.labelResolutionPends[:0] - ectx := m.ectx - bufPtr := m.c.BufPtr() for cur := root; cur != nil; cur = cur.next { offset := int64(len(*bufPtr)) if cur.kind == nop0 { l := cur.nop0Label() - if int(l) >= len(ectx.LabelPositions) { - continue - } - if pos := ectx.LabelPositions[l]; pos != nil { - pos.BinaryOffset = offset + pos := m.labelPositionPool.Get(int(l)) + if pos != nil { + pos.binaryOffset = offset } } @@ -1973,7 +2095,7 @@ func (m *machine) encodeWithoutSSA(root *instruction) { switch p.instr.kind { case jmp, jmpIf, lea: target := p.instr.jmpLabel() - targetOffset := ectx.LabelPositions[target].BinaryOffset + targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset imm32Offset := p.imm32Offset jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset)) @@ -1985,33 +2107,33 @@ func (m *machine) encodeWithoutSSA(root *instruction) { // Encode implements backend.Machine Encode. func (m *machine) Encode(ctx context.Context) (err error) { - ectx := m.ectx bufPtr := m.c.BufPtr() var fn string var fnIndex int - var labelToSSABlockID map[backend.Label]ssa.BasicBlockID + var labelPosToLabel map[*labelPosition]label if wazevoapi.PerfMapEnabled { fn = wazevoapi.GetCurrentFunctionName(ctx) - labelToSSABlockID = make(map[backend.Label]ssa.BasicBlockID) - for i, l := range ectx.SsaBlockIDToLabels { - labelToSSABlockID[l] = ssa.BasicBlockID(i) + labelPosToLabel = make(map[*labelPosition]label) + for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ { + pos := m.labelPositionPool.Get(i) + labelPosToLabel[pos] = label(i) } fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx) } m.labelResolutionPends = m.labelResolutionPends[:0] - for _, pos := range ectx.OrderedBlockLabels { + for _, pos := range m.orderedSSABlockLabelPos { offset := int64(len(*bufPtr)) - pos.BinaryOffset = offset - for cur := pos.Begin; cur != pos.End.next; cur = cur.next { + pos.binaryOffset = offset + for cur := pos.begin; cur != pos.end.next; cur = cur.next { offset := int64(len(*bufPtr)) switch cur.kind { case nop0: l := cur.nop0Label() - if pos := ectx.LabelPositions[l]; pos != nil { - pos.BinaryOffset = offset + if pos := m.labelPositionPool.Get(int(l)); pos != nil { + pos.binaryOffset = offset } case sourceOffsetInfo: m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo()) @@ -2026,22 +2148,16 @@ func (m *machine) Encode(ctx context.Context) (err error) { } if wazevoapi.PerfMapEnabled { - l := pos.L - var labelStr string - if blkID, ok := labelToSSABlockID[l]; ok { - labelStr = fmt.Sprintf("%s::SSA_Block[%s]", l, blkID) - } else { - labelStr = l.String() - } + l := labelPosToLabel[pos] size := int64(len(*bufPtr)) - offset - wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelStr)) + wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, l)) } } for i := range m.consts { offset := int64(len(*bufPtr)) vc := &m.consts[i] - vc.label.BinaryOffset = offset + vc.labelPos.binaryOffset = offset if vc._var == nil { lo, hi := vc.lo, vc.hi m.c.Emit8Bytes(lo) @@ -2059,7 +2175,7 @@ func (m *machine) Encode(ctx context.Context) (err error) { switch p.instr.kind { case jmp, jmpIf, lea, xmmUnaryRmR: target := p.instr.jmpLabel() - targetOffset := ectx.LabelPositions[target].BinaryOffset + targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset imm32Offset := p.imm32Offset jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset)) @@ -2068,7 +2184,7 @@ func (m *machine) Encode(ctx context.Context) (err error) { // Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes. targets := m.jmpTableTargets[p.instr.u1] for i, l := range targets { - targetOffset := ectx.LabelPositions[backend.Label(l)].BinaryOffset + targetOffset := m.labelPositionPool.Get(int(l)).binaryOffset jmpOffset := targetOffset - tableBegin binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset)) } @@ -2097,7 +2213,7 @@ func (m *machine) ResolveRelocations(refToBinaryOffset []int, binary []byte, rel // CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo. func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return } -func (m *machine) lowerIcmpToFlag(xd, yd *backend.SSAValueDefinition, _64 bool) { +func (m *machine) lowerIcmpToFlag(xd, yd backend.SSAValueDefinition, _64 bool) { x := m.getOperand_Reg(xd) y := m.getOperand_Mem_Imm32_Reg(yd) cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64) @@ -2140,7 +2256,7 @@ func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and boo // allocateInstr allocates an instruction. func (m *machine) allocateInstr() *instruction { - instr := m.ectx.InstructionPool.Allocate() + instr := m.instrPool.Allocate() if !m.regAllocStarted { instr.addedBeforeRegAlloc = true } @@ -2154,24 +2270,22 @@ func (m *machine) allocateNop() *instruction { } func (m *machine) insert(i *instruction) { - ectx := m.ectx - ectx.PendingInstructions = append(ectx.PendingInstructions, i) + m.pendingInstructions = append(m.pendingInstructions, i) } -func (m *machine) allocateBrTarget() (nop *instruction, l backend.Label) { //nolint - pos := m.allocateLabel() - l = pos.L +func (m *machine) allocateBrTarget() (nop *instruction, l label) { //nolint + l, pos := m.allocateLabel() nop = m.allocateInstr() nop.asNop0WithLabel(l) - pos.Begin, pos.End = nop, nop + pos.begin, pos.end = nop, nop return } -func (m *machine) allocateLabel() *labelPosition { - ectx := m.ectx - l := ectx.AllocateLabel() - pos := ectx.GetOrAllocateLabelPosition(l) - return pos +func (m *machine) allocateLabel() (label, *labelPosition) { + l := m.nextLabel + pos := m.labelPositionPool.GetOrAllocate(int(l)) + m.nextLabel++ + return l, pos } func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 { @@ -3185,22 +3299,22 @@ func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) { } } - xmaskLabel := m.allocateLabel() - m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xmaskLabel}) - ymaskLabel := m.allocateLabel() - m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: ymaskLabel}) + xl, xmaskPos := m.allocateLabel() + m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xl, labelPos: xmaskPos}) + yl, ymaskPos := m.allocateLabel() + m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: yl, labelPos: ymaskPos}) xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y)) tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg()) // Apply mask to X. tmp := m.c.AllocateVReg(ssa.TypeV128) - loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xmaskLabel.L)), tmp) + loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xl)), tmp) m.insert(loadMaskLo) m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX)) // Apply mask to Y. - loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(ymaskLabel.L)), tmp) + loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(yl)), tmp) m.insert(loadMaskHi) m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY)) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go index 8fa974c66..e53729860 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go @@ -12,7 +12,7 @@ func (m *machine) PostRegAlloc() { } func (m *machine) setupPrologue() { - cur := m.ectx.RootInstr + cur := m.rootInstr prevInitInst := cur.next // At this point, we have the stack layout as follows: @@ -130,14 +130,13 @@ func (m *machine) setupPrologue() { // 3. Inserts the dec/inc RSP instruction right before/after the call instruction. // 4. Lowering that is supposed to be done after regalloc. func (m *machine) postRegAlloc() { - ectx := m.ectx - for cur := ectx.RootInstr; cur != nil; cur = cur.next { + for cur := m.rootInstr; cur != nil; cur = cur.next { switch k := cur.kind; k { case ret: m.setupEpilogueAfter(cur.prev) continue case fcvtToSintSequence, fcvtToUintSequence: - m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] if k == fcvtToSintSequence { m.lowerFcvtToSintSequenceAfterRegalloc(cur) } else { @@ -146,29 +145,29 @@ func (m *machine) postRegAlloc() { prev := cur.prev next := cur.next cur := prev - for _, instr := range m.ectx.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } linkInstr(cur, next) continue case xmmCMov: - m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.lowerXmmCmovAfterRegAlloc(cur) prev := cur.prev next := cur.next cur := prev - for _, instr := range m.ectx.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } linkInstr(cur, next) continue case idivRemSequence: - m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.lowerIDivRemSequenceAfterRegAlloc(cur) prev := cur.prev next := cur.next cur := prev - for _, instr := range m.ectx.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } linkInstr(cur, next) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go index 0bb28ee9e..de9dcc944 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go @@ -1,13 +1,226 @@ package amd64 import ( - "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" ) -// InsertMoveBefore implements backend.RegAllocFunctionMachine. -func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { +// regAllocFn implements regalloc.Function. +type regAllocFn struct { + ssaB ssa.Builder + m *machine + loopNestingForestRoots []ssa.BasicBlock + blockIter int +} + +// PostOrderBlockIteratorBegin implements regalloc.Function. +func (f *regAllocFn) PostOrderBlockIteratorBegin() *labelPosition { + f.blockIter = len(f.m.orderedSSABlockLabelPos) - 1 + return f.PostOrderBlockIteratorNext() +} + +// PostOrderBlockIteratorNext implements regalloc.Function. +func (f *regAllocFn) PostOrderBlockIteratorNext() *labelPosition { + if f.blockIter < 0 { + return nil + } + b := f.m.orderedSSABlockLabelPos[f.blockIter] + f.blockIter-- + return b +} + +// ReversePostOrderBlockIteratorBegin implements regalloc.Function. +func (f *regAllocFn) ReversePostOrderBlockIteratorBegin() *labelPosition { + f.blockIter = 0 + return f.ReversePostOrderBlockIteratorNext() +} + +// ReversePostOrderBlockIteratorNext implements regalloc.Function. +func (f *regAllocFn) ReversePostOrderBlockIteratorNext() *labelPosition { + if f.blockIter >= len(f.m.orderedSSABlockLabelPos) { + return nil + } + b := f.m.orderedSSABlockLabelPos[f.blockIter] + f.blockIter++ + return b +} + +// ClobberedRegisters implements regalloc.Function. +func (f *regAllocFn) ClobberedRegisters(regs []regalloc.VReg) { + f.m.clobberedRegs = append(f.m.clobberedRegs[:0], regs...) +} + +// LoopNestingForestRoots implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestRoots() int { + f.loopNestingForestRoots = f.ssaB.LoopNestingForestRoots() + return len(f.loopNestingForestRoots) +} + +// LoopNestingForestRoot implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestRoot(i int) *labelPosition { + root := f.loopNestingForestRoots[i] + pos := f.m.getOrAllocateSSABlockLabelPosition(root) + return pos +} + +// LowestCommonAncestor implements regalloc.Function. +func (f *regAllocFn) LowestCommonAncestor(blk1, blk2 *labelPosition) *labelPosition { + sb := f.ssaB.LowestCommonAncestor(blk1.sb, blk2.sb) + pos := f.m.getOrAllocateSSABlockLabelPosition(sb) + return pos +} + +// Idom implements regalloc.Function. +func (f *regAllocFn) Idom(blk *labelPosition) *labelPosition { + sb := f.ssaB.Idom(blk.sb) + pos := f.m.getOrAllocateSSABlockLabelPosition(sb) + return pos +} + +// SwapBefore implements regalloc.Function. +func (f *regAllocFn) SwapBefore(x1, x2, tmp regalloc.VReg, instr *instruction) { + f.m.swap(instr.prev, x1, x2, tmp) +} + +// StoreRegisterBefore implements regalloc.Function. +func (f *regAllocFn) StoreRegisterBefore(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertStoreRegisterAt(v, instr, false) +} + +// StoreRegisterAfter implements regalloc.Function. +func (f *regAllocFn) StoreRegisterAfter(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertStoreRegisterAt(v, instr, true) +} + +// ReloadRegisterBefore implements regalloc.Function. +func (f *regAllocFn) ReloadRegisterBefore(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertReloadRegisterAt(v, instr, false) +} + +// ReloadRegisterAfter implements regalloc.Function. +func (f *regAllocFn) ReloadRegisterAfter(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertReloadRegisterAt(v, instr, true) +} + +// InsertMoveBefore implements regalloc.Function. +func (f *regAllocFn) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { + f.m.insertMoveBefore(dst, src, instr) +} + +// LoopNestingForestChild implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestChild(pos *labelPosition, i int) *labelPosition { + childSB := pos.sb.LoopNestingForestChildren()[i] + return f.m.getOrAllocateSSABlockLabelPosition(childSB) +} + +// Succ implements regalloc.Block. +func (f *regAllocFn) Succ(pos *labelPosition, i int) *labelPosition { + succSB := pos.sb.Succ(i) + if succSB.ReturnBlock() { + return nil + } + return f.m.getOrAllocateSSABlockLabelPosition(succSB) +} + +// Pred implements regalloc.Block. +func (f *regAllocFn) Pred(pos *labelPosition, i int) *labelPosition { + predSB := pos.sb.Pred(i) + return f.m.getOrAllocateSSABlockLabelPosition(predSB) +} + +// BlockParams implements regalloc.Function. +func (f *regAllocFn) BlockParams(pos *labelPosition, regs *[]regalloc.VReg) []regalloc.VReg { + c := f.m.c + *regs = (*regs)[:0] + for i := 0; i < pos.sb.Params(); i++ { + v := c.VRegOf(pos.sb.Param(i)) + *regs = append(*regs, v) + } + return *regs +} + +// ID implements regalloc.Block. +func (pos *labelPosition) ID() int32 { + return int32(pos.sb.ID()) +} + +// InstrIteratorBegin implements regalloc.Block. +func (pos *labelPosition) InstrIteratorBegin() *instruction { + ret := pos.begin + pos.cur = ret + return ret +} + +// InstrIteratorNext implements regalloc.Block. +func (pos *labelPosition) InstrIteratorNext() *instruction { + for { + if pos.cur == pos.end { + return nil + } + instr := pos.cur.next + pos.cur = instr + if instr == nil { + return nil + } else if instr.addedBeforeRegAlloc { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// InstrRevIteratorBegin implements regalloc.Block. +func (pos *labelPosition) InstrRevIteratorBegin() *instruction { + pos.cur = pos.end + return pos.cur +} + +// InstrRevIteratorNext implements regalloc.Block. +func (pos *labelPosition) InstrRevIteratorNext() *instruction { + for { + if pos.cur == pos.begin { + return nil + } + instr := pos.cur.prev + pos.cur = instr + if instr == nil { + return nil + } else if instr.addedBeforeRegAlloc { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// FirstInstr implements regalloc.Block. +func (pos *labelPosition) FirstInstr() *instruction { return pos.begin } + +// LastInstrForInsertion implements regalloc.Block. +func (pos *labelPosition) LastInstrForInsertion() *instruction { + return lastInstrForInsertion(pos.begin, pos.end) +} + +// Preds implements regalloc.Block. +func (pos *labelPosition) Preds() int { return pos.sb.Preds() } + +// Entry implements regalloc.Block. +func (pos *labelPosition) Entry() bool { return pos.sb.EntryBlock() } + +// Succs implements regalloc.Block. +func (pos *labelPosition) Succs() int { return pos.sb.Succs() } + +// LoopHeader implements regalloc.Block. +func (pos *labelPosition) LoopHeader() bool { return pos.sb.LoopHeader() } + +// LoopNestingForestChildren implements regalloc.Block. +func (pos *labelPosition) LoopNestingForestChildren() int { + return len(pos.sb.LoopNestingForestChildren()) +} + +func (m *machine) insertMoveBefore(dst, src regalloc.VReg, instr *instruction) { typ := src.RegType() if typ != dst.RegType() { panic("BUG: src and dst must have the same type") @@ -26,8 +239,7 @@ func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { linkInstr(cur, prevNext) } -// InsertStoreRegisterAt implements backend.RegAllocFunctionMachine. -func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { +func (m *machine) insertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { if !v.IsRealReg() { panic("BUG: VReg must be backed by real reg to be stored") } @@ -61,8 +273,7 @@ func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, aft return linkInstr(cur, prevNext) } -// InsertReloadRegisterAt implements backend.RegAllocFunctionMachine. -func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { +func (m *machine) insertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { if !v.IsRealReg() { panic("BUG: VReg must be backed by real reg to be stored") } @@ -98,13 +309,7 @@ func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, af return linkInstr(cur, prevNext) } -// ClobberedRegisters implements backend.RegAllocFunctionMachine. -func (m *machine) ClobberedRegisters(regs []regalloc.VReg) { - m.clobberedRegs = append(m.clobberedRegs[:0], regs...) -} - -// Swap implements backend.RegAllocFunctionMachine. -func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { +func (m *machine) swap(cur *instruction, x1, x2, tmp regalloc.VReg) { if x1.RegType() == regalloc.RegTypeInt { prevNext := cur.next xc := m.allocateInstr().asXCHG(x1, newOperandReg(x2), 8) @@ -113,25 +318,24 @@ func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { } else { if tmp.Valid() { prevNext := cur.next - m.InsertMoveBefore(tmp, x1, prevNext) - m.InsertMoveBefore(x1, x2, prevNext) - m.InsertMoveBefore(x2, tmp, prevNext) + m.insertMoveBefore(tmp, x1, prevNext) + m.insertMoveBefore(x1, x2, prevNext) + m.insertMoveBefore(x2, tmp, prevNext) } else { prevNext := cur.next r2 := x2.RealReg() // Temporarily spill x1 to stack. - cur = m.InsertStoreRegisterAt(x1, cur, true).prev + cur = m.insertStoreRegisterAt(x1, cur, true).prev // Then move x2 to x1. cur = linkInstr(cur, m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqa, newOperandReg(x2), x1)) linkInstr(cur, prevNext) // Then reload the original value on x1 from stack to r2. - m.InsertReloadRegisterAt(x1.SetRealReg(r2), cur, true) + m.insertReloadRegisterAt(x1.SetRealReg(r2), cur, true) } } } -// LastInstrForInsertion implements backend.RegAllocFunctionMachine. -func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction { +func lastInstrForInsertion(begin, end *instruction) *instruction { cur := end for cur.kind == nop0 { cur = cur.prev @@ -146,8 +350,3 @@ func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction { return end } } - -// SSABlockLabel implements backend.RegAllocFunctionMachine. -func (m *machine) SSABlockLabel(id ssa.BasicBlockID) backend.Label { - return m.ectx.SsaBlockIDToLabels[id] -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go index 539a8b754..8d514d857 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go @@ -127,7 +127,7 @@ func (m *machine) lowerSqmulRoundSat(x, y, ret ssa.Value) { tmpX := m.copyToTmp(xx.reg()) m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmulhrsw, yy, tmpX)) - m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmpX), tmp)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqw, newOperandReg(tmpX), tmp)) m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmpX)) m.copyTo(tmpX, m.c.VRegOf(ret)) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go index c6fcb8673..787975683 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go @@ -59,7 +59,7 @@ func (o *operand) format(_64 bool) string { case operandKindImm32: return fmt.Sprintf("$%d", int32(o.imm32())) case operandKindLabel: - return backend.Label(o.imm32()).String() + return label(o.imm32()).String() default: panic(fmt.Sprintf("BUG: invalid operand: %s", o.kind)) } @@ -85,22 +85,22 @@ func (o *operand) imm32() uint32 { return uint32(o.data) } -func (o *operand) label() backend.Label { +func (o *operand) label() label { switch o.kind { case operandKindLabel: - return backend.Label(o.data) + return label(o.data) case operandKindMem: mem := o.addressMode() if mem.kind() != amodeRipRel { panic("BUG: invalid label") } - return backend.Label(mem.imm32) + return label(mem.imm32) default: panic("BUG: invalid operand kind") } } -func newOperandLabel(label backend.Label) operand { +func newOperandLabel(label label) operand { return operand{kind: operandKindLabel, data: uint64(label)} } @@ -221,7 +221,7 @@ func (m *machine) newAmodeRegRegShift(imm32 uint32, base, index regalloc.VReg, s return ret } -func (m *machine) newAmodeRipRel(label backend.Label) *amode { +func (m *machine) newAmodeRipRel(label label) *amode { ret := m.amodePool.Allocate() *ret = amode{kindWithShift: uint32(amodeRipRel), imm32: uint32(label)} return ret @@ -246,18 +246,18 @@ func (a *amode) String() string { "%d(%s,%s,%d)", int32(a.imm32), formatVRegSized(a.base, true), formatVRegSized(a.index, true), shift) case amodeRipRel: - return fmt.Sprintf("%s(%%rip)", backend.Label(a.imm32)) + return fmt.Sprintf("%s(%%rip)", label(a.imm32)) default: panic("BUG: invalid amode kind") } } -func (m *machine) getOperand_Mem_Reg(def *backend.SSAValueDefinition) (op operand) { - if def.IsFromBlockParam() { - return newOperandReg(def.BlkParamVReg) +func (m *machine) getOperand_Mem_Reg(def backend.SSAValueDefinition) (op operand) { + if !def.IsFromInstr() { + return newOperandReg(m.c.VRegOf(def.V)) } - if def.SSAValue().Type() == ssa.TypeV128 { + if def.V.Type() == ssa.TypeV128 { // SIMD instructions require strict memory alignment, so we don't support the memory operand for V128 at the moment. return m.getOperand_Reg(def) } @@ -272,9 +272,9 @@ func (m *machine) getOperand_Mem_Reg(def *backend.SSAValueDefinition) (op operan return m.getOperand_Reg(def) } -func (m *machine) getOperand_Mem_Imm32_Reg(def *backend.SSAValueDefinition) (op operand) { - if def.IsFromBlockParam() { - return newOperandReg(def.BlkParamVReg) +func (m *machine) getOperand_Mem_Imm32_Reg(def backend.SSAValueDefinition) (op operand) { + if !def.IsFromInstr() { + return newOperandReg(m.c.VRegOf(def.V)) } if m.c.MatchInstr(def, ssa.OpcodeLoad) { @@ -287,9 +287,9 @@ func (m *machine) getOperand_Mem_Imm32_Reg(def *backend.SSAValueDefinition) (op return m.getOperand_Imm32_Reg(def) } -func (m *machine) getOperand_Imm32_Reg(def *backend.SSAValueDefinition) (op operand) { - if def.IsFromBlockParam() { - return newOperandReg(def.BlkParamVReg) +func (m *machine) getOperand_Imm32_Reg(def backend.SSAValueDefinition) (op operand) { + if !def.IsFromInstr() { + return newOperandReg(m.c.VRegOf(def.V)) } instr := def.Instr @@ -323,24 +323,14 @@ func asImm32(val uint64, allowSignExt bool) (uint32, bool) { return u32val, true } -func (m *machine) getOperand_Reg(def *backend.SSAValueDefinition) (op operand) { +func (m *machine) getOperand_Reg(def backend.SSAValueDefinition) (op operand) { var v regalloc.VReg - if def.IsFromBlockParam() { - v = def.BlkParamVReg + if instr := def.Instr; instr != nil && instr.Constant() { + // We inline all the constant instructions so that we could reduce the register usage. + v = m.lowerConstant(instr) + instr.MarkLowered() } else { - instr := def.Instr - if instr.Constant() { - // We inline all the constant instructions so that we could reduce the register usage. - v = m.lowerConstant(instr) - instr.MarkLowered() - } else { - if n := def.N; n == 0 { - v = m.c.VRegOf(instr.Return()) - } else { - _, rs := instr.Returns() - v = m.c.VRegOf(rs[n-1]) - } - } + v = m.c.VRegOf(def.V) } return newOperandReg(v) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go deleted file mode 100644 index 5219837e3..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !tinygo - -package amd64 - -import "reflect" - -// setSliceLimits sets both Cap and Len for the given reflected slice. -func setSliceLimits(s *reflect.SliceHeader, limit uintptr) { - s.Len = int(limit) - s.Cap = int(limit) -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go deleted file mode 100644 index df4cf46ec..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build tinygo - -package amd64 - -import "reflect" - -// setSliceLimits sets both Cap and Len for the given reflected slice. -func setSliceLimits(s *reflect.SliceHeader, limit uintptr) { - s.Len = limit - s.Len = limit -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go index 05ba5f027..ef823bdbd 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go @@ -9,12 +9,14 @@ ) func stackView(rbp, top uintptr) []byte { + l := int(top - rbp) var stackBuf []byte { - // TODO: use unsafe.Slice after floor version is set to Go 1.20. + //nolint:staticcheck hdr := (*reflect.SliceHeader)(unsafe.Pointer(&stackBuf)) hdr.Data = rbp - setSliceLimits(hdr, top-rbp) + hdr.Len = l + hdr.Cap = l } return stackBuf } @@ -72,9 +74,9 @@ func GoCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { // | SizeInBytes | // +-----------------+ <---- stackPointerBeforeGoCall // (low address) - data := unsafe.Pointer(uintptr(unsafe.Pointer(stackPointerBeforeGoCall)) + 8) + data := unsafe.Add(unsafe.Pointer(stackPointerBeforeGoCall), 8) size := *stackPointerBeforeGoCall / 8 - return unsafe.Slice((*uint64)(data), int(size)) + return unsafe.Slice((*uint64)(data), size) } func AdjustClonedStack(oldRsp, oldTop, rsp, rbp, top uintptr) { diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go index 4eaa13ce1..d1eaa7cd4 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go @@ -182,9 +182,9 @@ func (m *machine) LowerReturns(rets []ssa.Value) { // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the // caller side of the function call. -func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, slotBegin int64) { +func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, slotBegin int64) { arg := &a.Args[argIndex] - if def != nil && def.IsFromInstr() { + if def.IsFromInstr() { // Constant instructions are inlined. if inst := def.Instr; inst.Constant() { val := inst.Return() @@ -228,10 +228,9 @@ func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex i } func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) (*instruction, *addressMode) { - exct := m.executableContext - exct.PendingInstructions = exct.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] mode := m.resolveAddressModeForOffset(offset, dstBits, rn, allowTmpRegUse) - for _, instr := range exct.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } return cur, mode diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go index 99e6bb482..06f8a4a05 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go @@ -14,7 +14,6 @@ // CompileGoFunctionTrampoline implements backend.Machine. func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte { - exct := m.executableContext argBegin := 1 // Skips exec context by default. if needModuleContextPtr { argBegin++ @@ -26,7 +25,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * cur := m.allocateInstr() cur.asNop0() - exct.RootInstr = cur + m.rootInstr = cur // Execution context is always the first argument. execCtrPtr := x0VReg @@ -244,7 +243,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * ret.asRet() linkInstr(cur, ret) - m.encode(m.executableContext.RootInstr) + m.encode(m.rootInstr) return m.compiler.Buf() } @@ -302,20 +301,18 @@ func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []re } func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction { - exct := m.executableContext - exct.PendingInstructions = exct.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.lowerConstantI64(dst, v) - for _, instr := range exct.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } return cur } func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction { - exct := m.executableContext - exct.PendingInstructions = exct.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.lowerConstantI32(dst, v) - for _, instr := range exct.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } return cur diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go index 7121cb538..1f563428a 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go @@ -36,18 +36,6 @@ instructionKind byte ) -func asNop0(i *instruction) { - i.kind = nop0 -} - -func setNext(i, next *instruction) { - i.next = next -} - -func setPrev(i, prev *instruction) { - i.prev = prev -} - // IsCall implements regalloc.Instr IsCall. func (i *instruction) IsCall() bool { return i.kind == call @@ -63,21 +51,6 @@ func (i *instruction) IsReturn() bool { return i.kind == ret } -// Next implements regalloc.Instr Next. -func (i *instruction) Next() regalloc.Instr { - return i.next -} - -// Prev implements regalloc.Instr Prev. -func (i *instruction) Prev() regalloc.Instr { - return i.prev -} - -// AddedBeforeRegAlloc implements regalloc.Instr AddedBeforeRegAlloc. -func (i *instruction) AddedBeforeRegAlloc() bool { - return i.addedBeforeRegAlloc -} - type defKind byte const ( diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go index f0ede2d6a..21be9b71e 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go @@ -12,7 +12,7 @@ // Encode implements backend.Machine Encode. func (m *machine) Encode(ctx context.Context) error { m.resolveRelativeAddresses(ctx) - m.encode(m.executableContext.RootInstr) + m.encode(m.rootInstr) if l := len(m.compiler.Buf()); l > maxFunctionExecutableSize { return fmt.Errorf("function size exceeds the limit: %d > %d", l, maxFunctionExecutableSize) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go index 048bf3204..f9df356c0 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go @@ -17,19 +17,18 @@ // LowerSingleBranch implements backend.Machine. func (m *machine) LowerSingleBranch(br *ssa.Instruction) { - ectx := m.executableContext switch br.Opcode() { case ssa.OpcodeJump: - _, _, targetBlk := br.BranchData() + _, _, targetBlkID := br.BranchData() if br.IsFallthroughJump() { return } b := m.allocateInstr() - target := ectx.GetOrAllocateSSABlockLabel(targetBlk) - if target == labelReturn { + targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID) + if targetBlk.ReturnBlock() { b.asRet() } else { - b.asBr(target) + b.asBr(ssaBlockLabel(targetBlk)) } m.insert(b) case ssa.OpcodeBrTable: @@ -40,7 +39,8 @@ func (m *machine) LowerSingleBranch(br *ssa.Instruction) { } func (m *machine) lowerBrTable(i *ssa.Instruction) { - index, targets := i.BrTableData() + index, targetBlockIDs := i.BrTableData() + targetBlockCount := len(targetBlockIDs.View()) indexOperand := m.getOperand_NR(m.compiler.ValueDefinition(index), extModeNone) // Firstly, we have to do the bounds check of the index, and @@ -50,7 +50,7 @@ func (m *machine) lowerBrTable(i *ssa.Instruction) { // subs wzr, index, maxIndexReg // csel adjustedIndex, maxIndexReg, index, hs ;; if index is higher or equal than maxIndexReg. maxIndexReg := m.compiler.AllocateVReg(ssa.TypeI32) - m.lowerConstantI32(maxIndexReg, int32(len(targets)-1)) + m.lowerConstantI32(maxIndexReg, int32(targetBlockCount-1)) subs := m.allocateInstr() subs.asALU(aluOpSubS, xzrVReg, indexOperand, operandNR(maxIndexReg), false) m.insert(subs) @@ -61,24 +61,24 @@ func (m *machine) lowerBrTable(i *ssa.Instruction) { brSequence := m.allocateInstr() - tableIndex := m.addJmpTableTarget(targets) - brSequence.asBrTableSequence(adjustedIndex, tableIndex, len(targets)) + tableIndex := m.addJmpTableTarget(targetBlockIDs) + brSequence.asBrTableSequence(adjustedIndex, tableIndex, targetBlockCount) m.insert(brSequence) } // LowerConditionalBranch implements backend.Machine. func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { - exctx := m.executableContext - cval, args, targetBlk := b.BranchData() + cval, args, targetBlkID := b.BranchData() if len(args) > 0 { panic(fmt.Sprintf( "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", - exctx.CurrentSSABlk, - targetBlk, + m.currentLabelPos.sb, + targetBlkID, )) } - target := exctx.GetOrAllocateSSABlockLabel(targetBlk) + targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID) + target := ssaBlockLabel(targetBlk) cvalDef := m.compiler.ValueDefinition(cval) switch { @@ -791,7 +791,7 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) { default: panic("TODO: lowering " + op.String()) } - m.executableContext.FlushPendingInstructions() + m.FlushPendingInstructions() } func (m *machine) lowerShuffle(rd regalloc.VReg, rn, rm operand, lane1, lane2 uint64) { diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go index d9fbf1789..7a398c3d0 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go @@ -162,9 +162,9 @@ func (o operand) assignReg(v regalloc.VReg) operand { // // `mode` is used to extend the operand if the bit length is smaller than mode.bits(). // If the operand can be expressed as operandKindImm12, `mode` is ignored. -func (m *machine) getOperand_Imm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { - if def.IsFromBlockParam() { - return operandNR(def.BlkParamVReg) +func (m *machine) getOperand_Imm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { + if !def.IsFromInstr() { + return operandNR(m.compiler.VRegOf(def.V)) } instr := def.Instr @@ -179,9 +179,9 @@ func (m *machine) getOperand_Imm12_ER_SR_NR(def *backend.SSAValueDefinition, mod // getOperand_MaybeNegatedImm12_ER_SR_NR is almost the same as getOperand_Imm12_ER_SR_NR, but this might negate the immediate value. // If the immediate value is negated, the second return value is true, otherwise always false. -func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) { - if def.IsFromBlockParam() { - return operandNR(def.BlkParamVReg), false +func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) { + if !def.IsFromInstr() { + return operandNR(m.compiler.VRegOf(def.V)), false } instr := def.Instr @@ -193,7 +193,7 @@ func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDef } signExtended := int64(c) - if def.SSAValue().Type().Bits() == 32 { + if def.V.Type().Bits() == 32 { signExtended = (signExtended << 32) >> 32 } negatedWithoutSign := -signExtended @@ -208,9 +208,9 @@ func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDef // ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def). // // `mode` is used to extend the operand if the bit length is smaller than mode.bits(). -func (m *machine) getOperand_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { - if def.IsFromBlockParam() { - return operandNR(def.BlkParamVReg) +func (m *machine) getOperand_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { + if !def.IsFromInstr() { + return operandNR(m.compiler.VRegOf(def.V)) } if m.compiler.MatchInstr(def, ssa.OpcodeSExtend) || m.compiler.MatchInstr(def, ssa.OpcodeUExtend) { @@ -251,9 +251,9 @@ func (m *machine) getOperand_ER_SR_NR(def *backend.SSAValueDefinition, mode extM // ensureValueNR returns an operand of either operandKindSR or operandKindNR from the given value (defined by `def). // // `mode` is used to extend the operand if the bit length is smaller than mode.bits(). -func (m *machine) getOperand_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { - if def.IsFromBlockParam() { - return operandNR(def.BlkParamVReg) +func (m *machine) getOperand_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { + if !def.IsFromInstr() { + return operandNR(m.compiler.VRegOf(def.V)) } if m.compiler.MatchInstr(def, ssa.OpcodeIshl) { @@ -273,9 +273,9 @@ func (m *machine) getOperand_SR_NR(def *backend.SSAValueDefinition, mode extMode } // getOperand_ShiftImm_NR returns an operand of either operandKindShiftImm or operandKindNR from the given value (defined by `def). -func (m *machine) getOperand_ShiftImm_NR(def *backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) { - if def.IsFromBlockParam() { - return operandNR(def.BlkParamVReg) +func (m *machine) getOperand_ShiftImm_NR(def backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) { + if !def.IsFromInstr() { + return operandNR(m.compiler.VRegOf(def.V)) } instr := def.Instr @@ -289,28 +289,18 @@ func (m *machine) getOperand_ShiftImm_NR(def *backend.SSAValueDefinition, mode e // ensureValueNR returns an operand of operandKindNR from the given value (defined by `def). // // `mode` is used to extend the operand if the bit length is smaller than mode.bits(). -func (m *machine) getOperand_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { +func (m *machine) getOperand_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { var v regalloc.VReg - if def.IsFromBlockParam() { - v = def.BlkParamVReg + if def.IsFromInstr() && def.Instr.Constant() { + // We inline all the constant instructions so that we could reduce the register usage. + v = m.lowerConstant(def.Instr) + def.Instr.MarkLowered() } else { - instr := def.Instr - if instr.Constant() { - // We inline all the constant instructions so that we could reduce the register usage. - v = m.lowerConstant(instr) - instr.MarkLowered() - } else { - if n := def.N; n == 0 { - v = m.compiler.VRegOf(instr.Return()) - } else { - _, rs := instr.Returns() - v = m.compiler.VRegOf(rs[n-1]) - } - } + v = m.compiler.VRegOf(def.V) } r := v - switch inBits := def.SSAValue().Type().Bits(); { + switch inBits := def.V.Type().Bits(); { case mode == extModeNone: case inBits == 32 && (mode == extModeZeroExtend32 || mode == extModeSignExtend32): case inBits == 32 && mode == extModeZeroExtend64: diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go index 5f584f928..00e6b238f 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go @@ -3,6 +3,7 @@ import ( "context" "fmt" + "math" "strings" "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" @@ -14,12 +15,33 @@ type ( // machine implements backend.Machine. machine struct { - compiler backend.Compiler - executableContext *backend.ExecutableContextT[instruction] - currentABI *backend.FunctionABI + compiler backend.Compiler + currentABI *backend.FunctionABI + instrPool wazevoapi.Pool[instruction] + // labelPositionPool is the pool of labelPosition. The id is the label where + // if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID. + labelPositionPool wazevoapi.IDedPool[labelPosition] - regAlloc regalloc.Allocator - regAllocFn *backend.RegAllocFunction[*instruction, *machine] + // nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID + // so that we can have an identical label for the SSA block ID, which is useful for debugging. + nextLabel label + // rootInstr is the first instruction of the function. + rootInstr *instruction + // currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition. + currentLabelPos *labelPosition + // orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock. + orderedSSABlockLabelPos []*labelPosition + // returnLabelPos is the labelPosition for the return block. + returnLabelPos labelPosition + // perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock. + perBlockHead, perBlockEnd *instruction + // pendingInstructions are the instructions which are not yet emitted into the instruction list. + pendingInstructions []*instruction + // maxSSABlockID is the maximum ssa.BasicBlockID in the current function. + maxSSABlockID label + + regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn] + regAllocFn regAllocFn amodePool wazevoapi.Pool[addressMode] @@ -35,6 +57,8 @@ // jmpTableTargets holds the labels of the jump table targets. jmpTableTargets [][]uint32 + // jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table. + jmpTableTargetsNext int // spillSlotSize is the size of the stack slot in bytes used for spilling registers. // During the execution of the function, the stack looks like: @@ -91,45 +115,132 @@ nextLabel label offset int64 } +) - labelPosition = backend.LabelPosition[instruction] - label = backend.Label +type ( + // label represents a position in the generated code which is either + // a real instruction or the constant InstructionPool (e.g. jump tables). + // + // This is exactly the same as the traditional "label" in assembly code. + label uint32 + + // labelPosition represents the regions of the generated code which the label represents. + // This implements regalloc.Block. + labelPosition struct { + // sb is not nil if this corresponds to a ssa.BasicBlock. + sb ssa.BasicBlock + // cur is used to walk through the instructions in the block during the register allocation. + cur, + // begin and end are the first and last instructions of the block. + begin, end *instruction + // binaryOffset is the offset in the binary where the label is located. + binaryOffset int64 + } ) const ( - labelReturn = backend.LabelReturn - labelInvalid = backend.LabelInvalid + labelReturn label = math.MaxUint32 + labelInvalid = labelReturn - 1 ) +// String implements backend.Machine. +func (l label) String() string { + return fmt.Sprintf("L%d", l) +} + +func resetLabelPosition(l *labelPosition) { + *l = labelPosition{} +} + // NewBackend returns a new backend for arm64. func NewBackend() backend.Machine { m := &machine{ spillSlots: make(map[regalloc.VRegID]int64), - executableContext: newExecutableContext(), - regAlloc: regalloc.NewAllocator(regInfo), + regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo), amodePool: wazevoapi.NewPool[addressMode](resetAddressMode), + instrPool: wazevoapi.NewPool[instruction](resetInstruction), + labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition), } + m.regAllocFn.m = m return m } -func newExecutableContext() *backend.ExecutableContextT[instruction] { - return backend.NewExecutableContextT[instruction](resetInstruction, setNext, setPrev, asNop0) +func ssaBlockLabel(sb ssa.BasicBlock) label { + if sb.ReturnBlock() { + return labelReturn + } + return label(sb.ID()) } -// ExecutableContext implements backend.Machine. -func (m *machine) ExecutableContext() backend.ExecutableContext { - return m.executableContext +// getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block. +func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition { + if sb.ReturnBlock() { + m.returnLabelPos.sb = sb + return &m.returnLabelPos + } + + l := ssaBlockLabel(sb) + pos := m.labelPositionPool.GetOrAllocate(int(l)) + pos.sb = sb + return pos +} + +// LinkAdjacentBlocks implements backend.Machine. +func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { + prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next) + prevPos.end.next = nextPos.begin +} + +// StartBlock implements backend.Machine. +func (m *machine) StartBlock(blk ssa.BasicBlock) { + m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk) + labelPos := m.currentLabelPos + end := m.allocateNop() + m.perBlockHead, m.perBlockEnd = end, end + labelPos.begin, labelPos.end = end, end + m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos) +} + +// EndBlock implements ExecutableContext. +func (m *machine) EndBlock() { + // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions. + m.insertAtPerBlockHead(m.allocateNop()) + + m.currentLabelPos.begin = m.perBlockHead + + if m.currentLabelPos.sb.EntryBlock() { + m.rootInstr = m.perBlockHead + } +} + +func (m *machine) insertAtPerBlockHead(i *instruction) { + if m.perBlockHead == nil { + m.perBlockHead = i + m.perBlockEnd = i + return + } + + i.next = m.perBlockHead + m.perBlockHead.prev = i + m.perBlockHead = i +} + +// FlushPendingInstructions implements backend.Machine. +func (m *machine) FlushPendingInstructions() { + l := len(m.pendingInstructions) + if l == 0 { + return + } + for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order. + m.insertAtPerBlockHead(m.pendingInstructions[i]) + } + m.pendingInstructions = m.pendingInstructions[:0] } // RegAlloc implements backend.Machine Function. func (m *machine) RegAlloc() { - rf := m.regAllocFn - for _, pos := range m.executableContext.OrderedBlockLabels { - rf.AddBlock(pos.SB, pos.L, pos.Begin, pos.End) - } - m.regAllocStarted = true - m.regAlloc.DoAllocation(rf) + m.regAlloc.DoAllocation(&m.regAllocFn) // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes. m.spillSlotSize = (m.spillSlotSize + 15) &^ 15 } @@ -146,13 +257,22 @@ func (m *machine) Reset() { m.clobberedRegs = m.clobberedRegs[:0] m.regAllocStarted = false m.regAlloc.Reset() - m.regAllocFn.Reset() m.spillSlotSize = 0 m.unresolvedAddressModes = m.unresolvedAddressModes[:0] m.maxRequiredStackSizeForCalls = 0 - m.executableContext.Reset() - m.jmpTableTargets = m.jmpTableTargets[:0] + m.jmpTableTargetsNext = 0 m.amodePool.Reset() + m.instrPool.Reset() + m.labelPositionPool.Reset() + m.pendingInstructions = m.pendingInstructions[:0] + m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil + m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0] +} + +// StartLoweringFunction implements backend.Machine StartLoweringFunction. +func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) { + m.maxSSABlockID = label(maxBlockID) + m.nextLabel = label(maxBlockID) + 1 } // SetCurrentABI implements backend.Machine SetCurrentABI. @@ -168,12 +288,11 @@ func (m *machine) DisableStackCheck() { // SetCompiler implements backend.Machine. func (m *machine) SetCompiler(ctx backend.Compiler) { m.compiler = ctx - m.regAllocFn = backend.NewRegAllocFunction[*instruction, *machine](m, ctx.SSABuilder(), ctx) + m.regAllocFn.ssaB = ctx.SSABuilder() } func (m *machine) insert(i *instruction) { - ectx := m.executableContext - ectx.PendingInstructions = append(ectx.PendingInstructions, i) + m.pendingInstructions = append(m.pendingInstructions, i) } func (m *machine) insertBrTargetLabel() label { @@ -183,18 +302,18 @@ func (m *machine) insertBrTargetLabel() label { } func (m *machine) allocateBrTarget() (nop *instruction, l label) { - ectx := m.executableContext - l = ectx.AllocateLabel() + l = m.nextLabel + m.nextLabel++ nop = m.allocateInstr() nop.asNop0WithLabel(l) - pos := ectx.GetOrAllocateLabelPosition(l) - pos.Begin, pos.End = nop, nop + pos := m.labelPositionPool.GetOrAllocate(int(l)) + pos.begin, pos.end = nop, nop return } // allocateInstr allocates an instruction. func (m *machine) allocateInstr() *instruction { - instr := m.executableContext.InstructionPool.Allocate() + instr := m.instrPool.Allocate() if !m.regAllocStarted { instr.addedBeforeRegAlloc = true } @@ -251,7 +370,6 @@ func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruc // resolveRelativeAddresses resolves the relative addresses before encoding. func (m *machine) resolveRelativeAddresses(ctx context.Context) { - ectx := m.executableContext for { if len(m.unresolvedAddressModes) > 0 { arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP() @@ -265,35 +383,36 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { var fn string var fnIndex int - var labelToSSABlockID map[label]ssa.BasicBlockID + var labelPosToLabel map[*labelPosition]label if wazevoapi.PerfMapEnabled { - fn = wazevoapi.GetCurrentFunctionName(ctx) - labelToSSABlockID = make(map[label]ssa.BasicBlockID) - for i, l := range ectx.SsaBlockIDToLabels { - labelToSSABlockID[l] = ssa.BasicBlockID(i) + labelPosToLabel = make(map[*labelPosition]label) + for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ { + labelPosToLabel[m.labelPositionPool.Get(i)] = label(i) } + + fn = wazevoapi.GetCurrentFunctionName(ctx) fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx) } // Next, in order to determine the offsets of relative jumps, we have to calculate the size of each label. var offset int64 - for i, pos := range ectx.OrderedBlockLabels { - pos.BinaryOffset = offset + for i, pos := range m.orderedSSABlockLabelPos { + pos.binaryOffset = offset var size int64 - for cur := pos.Begin; ; cur = cur.next { + for cur := pos.begin; ; cur = cur.next { switch cur.kind { case nop0: l := cur.nop0Label() - if pos := ectx.LabelPositions[l]; pos != nil { - pos.BinaryOffset = offset + size + if pos := m.labelPositionPool.Get(int(l)); pos != nil { + pos.binaryOffset = offset + size } case condBr: if !cur.condBrOffsetResolved() { var nextLabel label - if i < len(ectx.OrderedBlockLabels)-1 { + if i < len(m.orderedSSABlockLabelPos)-1 { // Note: this is only used when the block ends with fallthrough, // therefore can be safely assumed that the next block exists when it's needed. - nextLabel = ectx.OrderedBlockLabels[i+1].L + nextLabel = ssaBlockLabel(m.orderedSSABlockLabelPos[i+1].sb) } m.condBrRelocs = append(m.condBrRelocs, condBrReloc{ cbr: cur, currentLabelPos: pos, offset: offset + size, @@ -302,21 +421,14 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { } } size += cur.size() - if cur == pos.End { + if cur == pos.end { break } } if wazevoapi.PerfMapEnabled { if size > 0 { - l := pos.L - var labelStr string - if blkID, ok := labelToSSABlockID[l]; ok { - labelStr = fmt.Sprintf("%s::SSA_Block[%s]", l, blkID) - } else { - labelStr = l.String() - } - wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelStr)) + wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelPosToLabel[pos])) } } offset += size @@ -330,7 +442,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { offset := reloc.offset target := cbr.condBrLabel() - offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset diff := offsetOfTarget - offset if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 { // This case the conditional branch is too huge. We place the trampoline instructions at the end of the current block, @@ -351,11 +463,11 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { } var currentOffset int64 - for cur := ectx.RootInstr; cur != nil; cur = cur.next { + for cur := m.rootInstr; cur != nil; cur = cur.next { switch cur.kind { case br: target := cur.brLabel() - offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset diff := offsetOfTarget - currentOffset divided := diff >> 2 if divided < minSignedInt26 || divided > maxSignedInt26 { @@ -366,7 +478,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { case condBr: if !cur.condBrOffsetResolved() { target := cur.condBrLabel() - offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + offsetOfTarget := m.labelPositionPool.Get(int(target)).binaryOffset diff := offsetOfTarget - currentOffset if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 { panic("BUG: branch relocation for large conditional branch larger than 19-bit range must be handled properly") @@ -378,7 +490,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { targets := m.jmpTableTargets[tableIndex] for i := range targets { l := label(targets[i]) - offsetOfTarget := ectx.LabelPositions[l].BinaryOffset + offsetOfTarget := m.labelPositionPool.Get(int(l)).binaryOffset diff := offsetOfTarget - (currentOffset + brTableSequenceOffsetTableBegin) targets[i] = uint32(diff) } @@ -399,7 +511,7 @@ func (m *machine) resolveRelativeAddresses(ctx context.Context) { ) func (m *machine) insertConditionalJumpTrampoline(cbr *instruction, currentBlk *labelPosition, nextLabel label) { - cur := currentBlk.End + cur := currentBlk.end originalTarget := cbr.condBrLabel() endNext := cur.next @@ -422,32 +534,27 @@ func (m *machine) insertConditionalJumpTrampoline(cbr *instruction, currentBlk * cur = linkInstr(cur, br) // Update the end of the current block. - currentBlk.End = cur + currentBlk.end = cur linkInstr(cur, endNext) } // Format implements backend.Machine. func (m *machine) Format() string { - ectx := m.executableContext begins := map[*instruction]label{} - for _, pos := range ectx.LabelPositions { + for l := label(0); l < m.nextLabel; l++ { + pos := m.labelPositionPool.Get(int(l)) if pos != nil { - begins[pos.Begin] = pos.L + begins[pos.begin] = l } } - irBlocks := map[label]ssa.BasicBlockID{} - for i, l := range ectx.SsaBlockIDToLabels { - irBlocks[l] = ssa.BasicBlockID(i) - } - var lines []string - for cur := ectx.RootInstr; cur != nil; cur = cur.next { + for cur := m.rootInstr; cur != nil; cur = cur.next { if l, ok := begins[cur]; ok { var labelStr string - if blkID, ok := irBlocks[l]; ok { - labelStr = fmt.Sprintf("%s (SSA Block: %s):", l, blkID) + if l <= m.maxSSABlockID { + labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, int(l)) } else { labelStr = fmt.Sprintf("%s:", l) } @@ -508,13 +615,17 @@ func (m *machine) frameSize() int64 { return s } -func (m *machine) addJmpTableTarget(targets []ssa.BasicBlock) (index int) { - // TODO: reuse the slice! - labels := make([]uint32, len(targets)) - for j, target := range targets { - labels[j] = uint32(m.executableContext.GetOrAllocateSSABlockLabel(target)) +func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) { + if m.jmpTableTargetsNext == len(m.jmpTableTargets) { + m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View()))) + } + + index = m.jmpTableTargetsNext + m.jmpTableTargetsNext++ + m.jmpTableTargets[index] = m.jmpTableTargets[index][:0] + for _, targetBlockID := range targets.View() { + target := m.compiler.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID)) + m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(target.ID())) } - index = len(m.jmpTableTargets) - m.jmpTableTargets = append(m.jmpTableTargets, labels) return } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go index d9032f921..c646a8fab 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go @@ -15,9 +15,7 @@ func (m *machine) PostRegAlloc() { // setupPrologue initializes the prologue of the function. func (m *machine) setupPrologue() { - ectx := m.executableContext - - cur := ectx.RootInstr + cur := m.rootInstr prevInitInst := cur.next // @@ -196,21 +194,20 @@ func (m *machine) createFrameSizeSlot(cur *instruction, s int64) *instruction { // 1. Removes the redundant copy instruction. // 2. Inserts the epilogue. func (m *machine) postRegAlloc() { - ectx := m.executableContext - for cur := ectx.RootInstr; cur != nil; cur = cur.next { + for cur := m.rootInstr; cur != nil; cur = cur.next { switch cur.kind { case ret: m.setupEpilogueAfter(cur.prev) case loadConstBlockArg: lc := cur next := lc.next - m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.lowerLoadConstantBlockArgAfterRegAlloc(lc) - for _, instr := range m.executableContext.PendingInstructions { + for _, instr := range m.pendingInstructions { cur = linkInstr(cur, instr) } linkInstr(cur, next) - m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] default: // Removes the redundant copy instruction. if cur.IsCopy() && cur.rn.realReg() == cur.rd.RealReg() { @@ -432,11 +429,9 @@ func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instructi // CompileStackGrowCallSequence implements backend.Machine. func (m *machine) CompileStackGrowCallSequence() []byte { - ectx := m.executableContext - cur := m.allocateInstr() cur.asNop0() - ectx.RootInstr = cur + m.rootInstr = cur // Save the callee saved and argument registers. cur = m.saveRegistersInExecutionContext(cur, saveRequiredRegs) @@ -458,16 +453,14 @@ func (m *machine) CompileStackGrowCallSequence() []byte { ret.asRet() linkInstr(cur, ret) - m.encode(ectx.RootInstr) + m.encode(m.rootInstr) return m.compiler.Buf() } func (m *machine) addsAddOrSubStackPointer(cur *instruction, rd regalloc.VReg, diff int64, add bool) *instruction { - ectx := m.executableContext - - ectx.PendingInstructions = ectx.PendingInstructions[:0] + m.pendingInstructions = m.pendingInstructions[:0] m.insertAddOrSubStackPointer(rd, diff, add) - for _, inserted := range ectx.PendingInstructions { + for _, inserted := range m.pendingInstructions { cur = linkInstr(cur, inserted) } return cur diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go index c7eb92cc2..f2ed53ae5 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go @@ -3,18 +3,226 @@ // This file implements the interfaces required for register allocations. See backend.RegAllocFunctionMachine. import ( - "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" ) -// ClobberedRegisters implements backend.RegAllocFunctionMachine. -func (m *machine) ClobberedRegisters(regs []regalloc.VReg) { - m.clobberedRegs = append(m.clobberedRegs[:0], regs...) +// regAllocFn implements regalloc.Function. +type regAllocFn struct { + ssaB ssa.Builder + m *machine + loopNestingForestRoots []ssa.BasicBlock + blockIter int } -// Swap implements backend.RegAllocFunctionMachine. -func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { +// PostOrderBlockIteratorBegin implements regalloc.Function. +func (f *regAllocFn) PostOrderBlockIteratorBegin() *labelPosition { + f.blockIter = len(f.m.orderedSSABlockLabelPos) - 1 + return f.PostOrderBlockIteratorNext() +} + +// PostOrderBlockIteratorNext implements regalloc.Function. +func (f *regAllocFn) PostOrderBlockIteratorNext() *labelPosition { + if f.blockIter < 0 { + return nil + } + b := f.m.orderedSSABlockLabelPos[f.blockIter] + f.blockIter-- + return b +} + +// ReversePostOrderBlockIteratorBegin implements regalloc.Function. +func (f *regAllocFn) ReversePostOrderBlockIteratorBegin() *labelPosition { + f.blockIter = 0 + return f.ReversePostOrderBlockIteratorNext() +} + +// ReversePostOrderBlockIteratorNext implements regalloc.Function. +func (f *regAllocFn) ReversePostOrderBlockIteratorNext() *labelPosition { + if f.blockIter >= len(f.m.orderedSSABlockLabelPos) { + return nil + } + b := f.m.orderedSSABlockLabelPos[f.blockIter] + f.blockIter++ + return b +} + +// ClobberedRegisters implements regalloc.Function. +func (f *regAllocFn) ClobberedRegisters(regs []regalloc.VReg) { + f.m.clobberedRegs = append(f.m.clobberedRegs[:0], regs...) +} + +// LoopNestingForestRoots implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestRoots() int { + f.loopNestingForestRoots = f.ssaB.LoopNestingForestRoots() + return len(f.loopNestingForestRoots) +} + +// LoopNestingForestRoot implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestRoot(i int) *labelPosition { + root := f.loopNestingForestRoots[i] + pos := f.m.getOrAllocateSSABlockLabelPosition(root) + return pos +} + +// LowestCommonAncestor implements regalloc.Function. +func (f *regAllocFn) LowestCommonAncestor(blk1, blk2 *labelPosition) *labelPosition { + sb := f.ssaB.LowestCommonAncestor(blk1.sb, blk2.sb) + pos := f.m.getOrAllocateSSABlockLabelPosition(sb) + return pos +} + +// Idom implements regalloc.Function. +func (f *regAllocFn) Idom(blk *labelPosition) *labelPosition { + sb := f.ssaB.Idom(blk.sb) + pos := f.m.getOrAllocateSSABlockLabelPosition(sb) + return pos +} + +// SwapBefore implements regalloc.Function. +func (f *regAllocFn) SwapBefore(x1, x2, tmp regalloc.VReg, instr *instruction) { + f.m.swap(instr.prev, x1, x2, tmp) +} + +// StoreRegisterBefore implements regalloc.Function. +func (f *regAllocFn) StoreRegisterBefore(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertStoreRegisterAt(v, instr, false) +} + +// StoreRegisterAfter implements regalloc.Function. +func (f *regAllocFn) StoreRegisterAfter(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertStoreRegisterAt(v, instr, true) +} + +// ReloadRegisterBefore implements regalloc.Function. +func (f *regAllocFn) ReloadRegisterBefore(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertReloadRegisterAt(v, instr, false) +} + +// ReloadRegisterAfter implements regalloc.Function. +func (f *regAllocFn) ReloadRegisterAfter(v regalloc.VReg, instr *instruction) { + m := f.m + m.insertReloadRegisterAt(v, instr, true) +} + +// InsertMoveBefore implements regalloc.Function. +func (f *regAllocFn) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { + f.m.insertMoveBefore(dst, src, instr) +} + +// LoopNestingForestChild implements regalloc.Function. +func (f *regAllocFn) LoopNestingForestChild(pos *labelPosition, i int) *labelPosition { + childSB := pos.sb.LoopNestingForestChildren()[i] + return f.m.getOrAllocateSSABlockLabelPosition(childSB) +} + +// Succ implements regalloc.Block. +func (f *regAllocFn) Succ(pos *labelPosition, i int) *labelPosition { + succSB := pos.sb.Succ(i) + if succSB.ReturnBlock() { + return nil + } + return f.m.getOrAllocateSSABlockLabelPosition(succSB) +} + +// Pred implements regalloc.Block. +func (f *regAllocFn) Pred(pos *labelPosition, i int) *labelPosition { + predSB := pos.sb.Pred(i) + return f.m.getOrAllocateSSABlockLabelPosition(predSB) +} + +// BlockParams implements regalloc.Function. +func (f *regAllocFn) BlockParams(pos *labelPosition, regs *[]regalloc.VReg) []regalloc.VReg { + c := f.m.compiler + *regs = (*regs)[:0] + for i := 0; i < pos.sb.Params(); i++ { + v := c.VRegOf(pos.sb.Param(i)) + *regs = append(*regs, v) + } + return *regs +} + +// ID implements regalloc.Block. +func (pos *labelPosition) ID() int32 { + return int32(pos.sb.ID()) +} + +// InstrIteratorBegin implements regalloc.Block. +func (pos *labelPosition) InstrIteratorBegin() *instruction { + ret := pos.begin + pos.cur = ret + return ret +} + +// InstrIteratorNext implements regalloc.Block. +func (pos *labelPosition) InstrIteratorNext() *instruction { + for { + if pos.cur == pos.end { + return nil + } + instr := pos.cur.next + pos.cur = instr + if instr == nil { + return nil + } else if instr.addedBeforeRegAlloc { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// InstrRevIteratorBegin implements regalloc.Block. +func (pos *labelPosition) InstrRevIteratorBegin() *instruction { + pos.cur = pos.end + return pos.cur +} + +// InstrRevIteratorNext implements regalloc.Block. +func (pos *labelPosition) InstrRevIteratorNext() *instruction { + for { + if pos.cur == pos.begin { + return nil + } + instr := pos.cur.prev + pos.cur = instr + if instr == nil { + return nil + } else if instr.addedBeforeRegAlloc { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// FirstInstr implements regalloc.Block. +func (pos *labelPosition) FirstInstr() *instruction { return pos.begin } + +// LastInstrForInsertion implements regalloc.Block. +func (pos *labelPosition) LastInstrForInsertion() *instruction { + return lastInstrForInsertion(pos.begin, pos.end) +} + +// Preds implements regalloc.Block. +func (pos *labelPosition) Preds() int { return pos.sb.Preds() } + +// Entry implements regalloc.Block. +func (pos *labelPosition) Entry() bool { return pos.sb.EntryBlock() } + +// Succs implements regalloc.Block. +func (pos *labelPosition) Succs() int { return pos.sb.Succs() } + +// LoopHeader implements regalloc.Block. +func (pos *labelPosition) LoopHeader() bool { return pos.sb.LoopHeader() } + +// LoopNestingForestChildren implements regalloc.Block. +func (pos *labelPosition) LoopNestingForestChildren() int { + return len(pos.sb.LoopNestingForestChildren()) +} + +func (m *machine) swap(cur *instruction, x1, x2, tmp regalloc.VReg) { prevNext := cur.next var mov1, mov2, mov3 *instruction if x1.RegType() == regalloc.RegTypeInt { @@ -32,12 +240,12 @@ func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { if !tmp.Valid() { r2 := x2.RealReg() // Temporarily spill x1 to stack. - cur = m.InsertStoreRegisterAt(x1, cur, true).prev + cur = m.insertStoreRegisterAt(x1, cur, true).prev // Then move x2 to x1. cur = linkInstr(cur, m.allocateInstr().asFpuMov128(x1, x2)) linkInstr(cur, prevNext) // Then reload the original value on x1 from stack to r2. - m.InsertReloadRegisterAt(x1.SetRealReg(r2), cur, true) + m.insertReloadRegisterAt(x1.SetRealReg(r2), cur, true) } else { mov1 = m.allocateInstr().asFpuMov128(tmp, x1) mov2 = m.allocateInstr().asFpuMov128(x1, x2) @@ -50,8 +258,7 @@ func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { } } -// InsertMoveBefore implements backend.RegAllocFunctionMachine. -func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { +func (m *machine) insertMoveBefore(dst, src regalloc.VReg, instr *instruction) { typ := src.RegType() if typ != dst.RegType() { panic("BUG: src and dst must have the same type") @@ -70,13 +277,7 @@ func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { linkInstr(cur, prevNext) } -// SSABlockLabel implements backend.RegAllocFunctionMachine. -func (m *machine) SSABlockLabel(id ssa.BasicBlockID) backend.Label { - return m.executableContext.SsaBlockIDToLabels[id] -} - -// InsertStoreRegisterAt implements backend.RegAllocFunctionMachine. -func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { +func (m *machine) insertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { if !v.IsRealReg() { panic("BUG: VReg must be backed by real reg to be stored") } @@ -100,8 +301,7 @@ func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, aft return linkInstr(cur, prevNext) } -// InsertReloadRegisterAt implements backend.RegAllocFunctionMachine. -func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { +func (m *machine) insertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { if !v.IsRealReg() { panic("BUG: VReg must be backed by real reg to be stored") } @@ -134,8 +334,7 @@ func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, af return linkInstr(cur, prevNext) } -// LastInstrForInsertion implements backend.RegAllocFunctionMachine. -func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction { +func lastInstrForInsertion(begin, end *instruction) *instruction { cur := end for cur.kind == nop0 { cur = cur.prev diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go index edb0e36e3..a72b86f6b 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go @@ -14,7 +14,7 @@ func UnwindStack(sp, _, top uintptr, returnAddresses []uintptr) []uintptr { var stackBuf []byte { - // TODO: use unsafe.Slice after floor version is set to Go 1.20. + //nolint:staticcheck hdr := (*reflect.SliceHeader)(unsafe.Pointer(&stackBuf)) hdr.Data = sp hdr.Len = l @@ -78,13 +78,7 @@ func GoCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { // +-----------------+ <---- stackPointerBeforeGoCall // (low address) ptr := unsafe.Pointer(stackPointerBeforeGoCall) + data := (*uint64)(unsafe.Add(ptr, 16)) // skips the (frame_size, sliceSize). size := *(*uint64)(unsafe.Add(ptr, 8)) - var view []uint64 - { - sh := (*reflect.SliceHeader)(unsafe.Pointer(&view)) - sh.Data = uintptr(unsafe.Add(ptr, 16)) // skips the (frame_size, sliceSize). - sh.Len = int(size) - sh.Cap = int(size) - } - return view + return unsafe.Slice(data, size) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go index 54ce89e46..9044a9e4b 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go @@ -11,7 +11,24 @@ type ( // Machine is a backend for a specific ISA machine. Machine interface { - ExecutableContext() ExecutableContext + // StartLoweringFunction is called when the compilation of the given function is started. + // The maxBlockID is the maximum ssa.BasicBlockID in the function. + StartLoweringFunction(maxBlockID ssa.BasicBlockID) + + // LinkAdjacentBlocks is called after finished lowering all blocks in order to create one single instruction list. + LinkAdjacentBlocks(prev, next ssa.BasicBlock) + + // StartBlock is called when the compilation of the given block is started. + // The order of this being called is the reverse post order of the ssa.BasicBlock(s) as we iterate with + // ssa.Builder BlockIteratorReversePostOrderBegin and BlockIteratorReversePostOrderEnd. + StartBlock(ssa.BasicBlock) + + // EndBlock is called when the compilation of the current block is finished. + EndBlock() + + // FlushPendingInstructions flushes the pending instructions to the buffer. + // This will be called after the lowering of each SSA Instruction. + FlushPendingInstructions() // DisableStackCheck disables the stack check for the current compilation for debugging/testing. DisableStackCheck() diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go deleted file mode 100644 index 655370786..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go +++ /dev/null @@ -1,321 +0,0 @@ -package backend - -import ( - "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" - "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" -) - -// RegAllocFunctionMachine is the interface for the machine specific logic that will be used in RegAllocFunction. -type RegAllocFunctionMachine[I regalloc.InstrConstraint] interface { - // InsertMoveBefore inserts the move instruction from src to dst before the given instruction. - InsertMoveBefore(dst, src regalloc.VReg, instr I) - // InsertStoreRegisterAt inserts the instruction(s) to store the given virtual register at the given instruction. - // If after is true, the instruction(s) will be inserted after the given instruction, otherwise before. - InsertStoreRegisterAt(v regalloc.VReg, instr I, after bool) I - // InsertReloadRegisterAt inserts the instruction(s) to reload the given virtual register at the given instruction. - // If after is true, the instruction(s) will be inserted after the given instruction, otherwise before. - InsertReloadRegisterAt(v regalloc.VReg, instr I, after bool) I - // ClobberedRegisters is called when the register allocation is done and the clobbered registers are known. - ClobberedRegisters(regs []regalloc.VReg) - // Swap swaps the two virtual registers after the given instruction. - Swap(cur I, x1, x2, tmp regalloc.VReg) - // LastInstrForInsertion implements LastInstrForInsertion of regalloc.Function. See its comment for details. - LastInstrForInsertion(begin, end I) I - // SSABlockLabel returns the label of the given ssa.BasicBlockID. - SSABlockLabel(id ssa.BasicBlockID) Label -} - -type ( - // RegAllocFunction implements regalloc.Function. - RegAllocFunction[I regalloc.InstrConstraint, m RegAllocFunctionMachine[I]] struct { - m m - ssb ssa.Builder - c Compiler - // iter is the iterator for reversePostOrderBlocks - iter int - reversePostOrderBlocks []RegAllocBlock[I, m] - // labelToRegAllocBlockIndex maps label to the index of reversePostOrderBlocks. - labelToRegAllocBlockIndex [] /* Label to */ int - loopNestingForestRoots []ssa.BasicBlock - } - - // RegAllocBlock implements regalloc.Block. - RegAllocBlock[I regalloc.InstrConstraint, m RegAllocFunctionMachine[I]] struct { - // f is the function this instruction belongs to. Used to reuse the regAllocFunctionImpl.predsSlice slice for Defs() and Uses(). - f *RegAllocFunction[I, m] - sb ssa.BasicBlock - l Label - begin, end I - loopNestingForestChildren []ssa.BasicBlock - cur I - id int - cachedLastInstrForInsertion I - } -) - -// NewRegAllocFunction returns a new RegAllocFunction. -func NewRegAllocFunction[I regalloc.InstrConstraint, M RegAllocFunctionMachine[I]](m M, ssb ssa.Builder, c Compiler) *RegAllocFunction[I, M] { - return &RegAllocFunction[I, M]{ - m: m, - ssb: ssb, - c: c, - } -} - -// AddBlock adds a new block to the function. -func (f *RegAllocFunction[I, M]) AddBlock(sb ssa.BasicBlock, l Label, begin, end I) { - i := len(f.reversePostOrderBlocks) - f.reversePostOrderBlocks = append(f.reversePostOrderBlocks, RegAllocBlock[I, M]{ - f: f, - sb: sb, - l: l, - begin: begin, - end: end, - id: int(sb.ID()), - }) - if len(f.labelToRegAllocBlockIndex) <= int(l) { - f.labelToRegAllocBlockIndex = append(f.labelToRegAllocBlockIndex, make([]int, int(l)-len(f.labelToRegAllocBlockIndex)+1)...) - } - f.labelToRegAllocBlockIndex[l] = i -} - -// Reset resets the function for the next compilation. -func (f *RegAllocFunction[I, M]) Reset() { - f.reversePostOrderBlocks = f.reversePostOrderBlocks[:0] - f.iter = 0 -} - -// StoreRegisterAfter implements regalloc.Function StoreRegisterAfter. -func (f *RegAllocFunction[I, M]) StoreRegisterAfter(v regalloc.VReg, instr regalloc.Instr) { - m := f.m - m.InsertStoreRegisterAt(v, instr.(I), true) -} - -// ReloadRegisterBefore implements regalloc.Function ReloadRegisterBefore. -func (f *RegAllocFunction[I, M]) ReloadRegisterBefore(v regalloc.VReg, instr regalloc.Instr) { - m := f.m - m.InsertReloadRegisterAt(v, instr.(I), false) -} - -// ReloadRegisterAfter implements regalloc.Function ReloadRegisterAfter. -func (f *RegAllocFunction[I, M]) ReloadRegisterAfter(v regalloc.VReg, instr regalloc.Instr) { - m := f.m - m.InsertReloadRegisterAt(v, instr.(I), true) -} - -// StoreRegisterBefore implements regalloc.Function StoreRegisterBefore. -func (f *RegAllocFunction[I, M]) StoreRegisterBefore(v regalloc.VReg, instr regalloc.Instr) { - m := f.m - m.InsertStoreRegisterAt(v, instr.(I), false) -} - -// ClobberedRegisters implements regalloc.Function ClobberedRegisters. -func (f *RegAllocFunction[I, M]) ClobberedRegisters(regs []regalloc.VReg) { - f.m.ClobberedRegisters(regs) -} - -// SwapBefore implements regalloc.Function SwapBefore. -func (f *RegAllocFunction[I, M]) SwapBefore(x1, x2, tmp regalloc.VReg, instr regalloc.Instr) { - f.m.Swap(instr.Prev().(I), x1, x2, tmp) -} - -// PostOrderBlockIteratorBegin implements regalloc.Function PostOrderBlockIteratorBegin. -func (f *RegAllocFunction[I, M]) PostOrderBlockIteratorBegin() regalloc.Block { - f.iter = len(f.reversePostOrderBlocks) - 1 - return f.PostOrderBlockIteratorNext() -} - -// PostOrderBlockIteratorNext implements regalloc.Function PostOrderBlockIteratorNext. -func (f *RegAllocFunction[I, M]) PostOrderBlockIteratorNext() regalloc.Block { - if f.iter < 0 { - return nil - } - b := &f.reversePostOrderBlocks[f.iter] - f.iter-- - return b -} - -// ReversePostOrderBlockIteratorBegin implements regalloc.Function ReversePostOrderBlockIteratorBegin. -func (f *RegAllocFunction[I, M]) ReversePostOrderBlockIteratorBegin() regalloc.Block { - f.iter = 0 - return f.ReversePostOrderBlockIteratorNext() -} - -// ReversePostOrderBlockIteratorNext implements regalloc.Function ReversePostOrderBlockIteratorNext. -func (f *RegAllocFunction[I, M]) ReversePostOrderBlockIteratorNext() regalloc.Block { - if f.iter >= len(f.reversePostOrderBlocks) { - return nil - } - b := &f.reversePostOrderBlocks[f.iter] - f.iter++ - return b -} - -// LoopNestingForestRoots implements regalloc.Function LoopNestingForestRoots. -func (f *RegAllocFunction[I, M]) LoopNestingForestRoots() int { - f.loopNestingForestRoots = f.ssb.LoopNestingForestRoots() - return len(f.loopNestingForestRoots) -} - -// LoopNestingForestRoot implements regalloc.Function LoopNestingForestRoot. -func (f *RegAllocFunction[I, M]) LoopNestingForestRoot(i int) regalloc.Block { - blk := f.loopNestingForestRoots[i] - l := f.m.SSABlockLabel(blk.ID()) - index := f.labelToRegAllocBlockIndex[l] - return &f.reversePostOrderBlocks[index] -} - -// InsertMoveBefore implements regalloc.Function InsertMoveBefore. -func (f *RegAllocFunction[I, M]) InsertMoveBefore(dst, src regalloc.VReg, instr regalloc.Instr) { - f.m.InsertMoveBefore(dst, src, instr.(I)) -} - -// LowestCommonAncestor implements regalloc.Function LowestCommonAncestor. -func (f *RegAllocFunction[I, M]) LowestCommonAncestor(blk1, blk2 regalloc.Block) regalloc.Block { - ret := f.ssb.LowestCommonAncestor(blk1.(*RegAllocBlock[I, M]).sb, blk2.(*RegAllocBlock[I, M]).sb) - l := f.m.SSABlockLabel(ret.ID()) - index := f.labelToRegAllocBlockIndex[l] - return &f.reversePostOrderBlocks[index] -} - -// Idom implements regalloc.Function Idom. -func (f *RegAllocFunction[I, M]) Idom(blk regalloc.Block) regalloc.Block { - builder := f.ssb - idom := builder.Idom(blk.(*RegAllocBlock[I, M]).sb) - if idom == nil { - panic("BUG: idom must not be nil") - } - l := f.m.SSABlockLabel(idom.ID()) - index := f.labelToRegAllocBlockIndex[l] - return &f.reversePostOrderBlocks[index] -} - -// ID implements regalloc.Block. -func (r *RegAllocBlock[I, m]) ID() int32 { return int32(r.id) } - -// BlockParams implements regalloc.Block. -func (r *RegAllocBlock[I, m]) BlockParams(regs *[]regalloc.VReg) []regalloc.VReg { - c := r.f.c - *regs = (*regs)[:0] - for i := 0; i < r.sb.Params(); i++ { - v := c.VRegOf(r.sb.Param(i)) - *regs = append(*regs, v) - } - return *regs -} - -// InstrIteratorBegin implements regalloc.Block. -func (r *RegAllocBlock[I, m]) InstrIteratorBegin() regalloc.Instr { - r.cur = r.begin - return r.cur -} - -// InstrIteratorNext implements regalloc.Block. -func (r *RegAllocBlock[I, m]) InstrIteratorNext() regalloc.Instr { - for { - if r.cur == r.end { - return nil - } - instr := r.cur.Next() - r.cur = instr.(I) - if instr == nil { - return nil - } else if instr.AddedBeforeRegAlloc() { - // Only concerned about the instruction added before regalloc. - return instr - } - } -} - -// InstrRevIteratorBegin implements regalloc.Block. -func (r *RegAllocBlock[I, m]) InstrRevIteratorBegin() regalloc.Instr { - r.cur = r.end - return r.cur -} - -// InstrRevIteratorNext implements regalloc.Block. -func (r *RegAllocBlock[I, m]) InstrRevIteratorNext() regalloc.Instr { - for { - if r.cur == r.begin { - return nil - } - instr := r.cur.Prev() - r.cur = instr.(I) - if instr == nil { - return nil - } else if instr.AddedBeforeRegAlloc() { - // Only concerned about the instruction added before regalloc. - return instr - } - } -} - -// FirstInstr implements regalloc.Block. -func (r *RegAllocBlock[I, m]) FirstInstr() regalloc.Instr { - return r.begin -} - -// EndInstr implements regalloc.Block. -func (r *RegAllocBlock[I, m]) EndInstr() regalloc.Instr { - return r.end -} - -// LastInstrForInsertion implements regalloc.Block. -func (r *RegAllocBlock[I, m]) LastInstrForInsertion() regalloc.Instr { - var nil I - if r.cachedLastInstrForInsertion == nil { - r.cachedLastInstrForInsertion = r.f.m.LastInstrForInsertion(r.begin, r.end) - } - return r.cachedLastInstrForInsertion -} - -// Preds implements regalloc.Block. -func (r *RegAllocBlock[I, m]) Preds() int { return r.sb.Preds() } - -// Pred implements regalloc.Block. -func (r *RegAllocBlock[I, m]) Pred(i int) regalloc.Block { - sb := r.sb - pred := sb.Pred(i) - l := r.f.m.SSABlockLabel(pred.ID()) - index := r.f.labelToRegAllocBlockIndex[l] - return &r.f.reversePostOrderBlocks[index] -} - -// Entry implements regalloc.Block. -func (r *RegAllocBlock[I, m]) Entry() bool { return r.sb.EntryBlock() } - -// Succs implements regalloc.Block. -func (r *RegAllocBlock[I, m]) Succs() int { - return r.sb.Succs() -} - -// Succ implements regalloc.Block. -func (r *RegAllocBlock[I, m]) Succ(i int) regalloc.Block { - sb := r.sb - succ := sb.Succ(i) - if succ.ReturnBlock() { - return nil - } - l := r.f.m.SSABlockLabel(succ.ID()) - index := r.f.labelToRegAllocBlockIndex[l] - return &r.f.reversePostOrderBlocks[index] -} - -// LoopHeader implements regalloc.Block. -func (r *RegAllocBlock[I, m]) LoopHeader() bool { - return r.sb.LoopHeader() -} - -// LoopNestingForestChildren implements regalloc.Block. -func (r *RegAllocBlock[I, m]) LoopNestingForestChildren() int { - r.loopNestingForestChildren = r.sb.LoopNestingForestChildren() - return len(r.loopNestingForestChildren) -} - -// LoopNestingForestChild implements regalloc.Block. -func (r *RegAllocBlock[I, m]) LoopNestingForestChild(i int) regalloc.Block { - blk := r.loopNestingForestChildren[i] - l := r.f.m.SSABlockLabel(blk.ID()) - index := r.f.labelToRegAllocBlockIndex[l] - return &r.f.reversePostOrderBlocks[index] -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go index 23157b478..5d15bd9dc 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go @@ -4,104 +4,100 @@ // These interfaces are implemented by ISA-specific backends to abstract away the details, and allow the register // allocators to work on any ISA. -// -// TODO: the interfaces are not stabilized yet, especially x64 will need some changes. E.g. x64 has an addressing mode -// where index can be in memory. That kind of info will be useful to reduce the register pressure, and should be leveraged -// by the register allocators, like https://docs.rs/regalloc2/latest/regalloc2/enum.OperandConstraint.html type ( // Function is the top-level interface to do register allocation, which corresponds to a CFG containing // Blocks(s). - Function interface { + // + // I is the type of the instruction, and B is the type of the basic block. + Function[I Instr, B Block[I]] interface { // PostOrderBlockIteratorBegin returns the first block in the post-order traversal of the CFG. // In other words, the last blocks in the CFG will be returned first. - PostOrderBlockIteratorBegin() Block + PostOrderBlockIteratorBegin() B // PostOrderBlockIteratorNext returns the next block in the post-order traversal of the CFG. - PostOrderBlockIteratorNext() Block + PostOrderBlockIteratorNext() B // ReversePostOrderBlockIteratorBegin returns the first block in the reverse post-order traversal of the CFG. // In other words, the first blocks in the CFG will be returned first. - ReversePostOrderBlockIteratorBegin() Block + ReversePostOrderBlockIteratorBegin() B // ReversePostOrderBlockIteratorNext returns the next block in the reverse post-order traversal of the CFG. - ReversePostOrderBlockIteratorNext() Block + ReversePostOrderBlockIteratorNext() B // ClobberedRegisters tell the clobbered registers by this function. ClobberedRegisters([]VReg) // LoopNestingForestRoots returns the number of roots of the loop nesting forest in a function. LoopNestingForestRoots() int // LoopNestingForestRoot returns the i-th root of the loop nesting forest in a function. - LoopNestingForestRoot(i int) Block + LoopNestingForestRoot(i int) B // LowestCommonAncestor returns the lowest common ancestor of two blocks in the dominator tree. - LowestCommonAncestor(blk1, blk2 Block) Block + LowestCommonAncestor(blk1, blk2 B) B // Idom returns the immediate dominator of the given block. - Idom(blk Block) Block + Idom(blk B) B + + // LoopNestingForestChild returns the i-th child of the block in the loop nesting forest. + LoopNestingForestChild(b B, i int) B + // Pred returns the i-th predecessor of the block in the CFG. + Pred(b B, i int) B + // Succ returns the i-th successor of the block in the CFG. + Succ(b B, i int) B + // BlockParams returns the virtual registers used as the parameters of this block. + BlockParams(B, *[]VReg) []VReg // Followings are for rewriting the function. - // SwapAtEndOfBlock swaps the two virtual registers at the end of the given block. - SwapBefore(x1, x2, tmp VReg, instr Instr) + // SwapBefore swaps the two virtual registers at the end of the given block. + SwapBefore(x1, x2, tmp VReg, instr I) // StoreRegisterBefore inserts store instruction(s) before the given instruction for the given virtual register. - StoreRegisterBefore(v VReg, instr Instr) + StoreRegisterBefore(v VReg, instr I) // StoreRegisterAfter inserts store instruction(s) after the given instruction for the given virtual register. - StoreRegisterAfter(v VReg, instr Instr) + StoreRegisterAfter(v VReg, instr I) // ReloadRegisterBefore inserts reload instruction(s) before the given instruction for the given virtual register. - ReloadRegisterBefore(v VReg, instr Instr) + ReloadRegisterBefore(v VReg, instr I) // ReloadRegisterAfter inserts reload instruction(s) after the given instruction for the given virtual register. - ReloadRegisterAfter(v VReg, instr Instr) + ReloadRegisterAfter(v VReg, instr I) // InsertMoveBefore inserts move instruction(s) before the given instruction for the given virtual registers. - InsertMoveBefore(dst, src VReg, instr Instr) + InsertMoveBefore(dst, src VReg, instr I) } // Block is a basic block in the CFG of a function, and it consists of multiple instructions, and predecessor Block(s). - Block interface { + // Right now, this corresponds to a ssa.BasicBlock lowered to the machine level. + Block[I Instr] interface { + comparable // ID returns the unique identifier of this block which is ordered in the reverse post-order traversal of the CFG. ID() int32 - // BlockParams returns the virtual registers used as the parameters of this block. - BlockParams(*[]VReg) []VReg // InstrIteratorBegin returns the first instruction in this block. Instructions added after lowering must be skipped. // Note: multiple Instr(s) will not be held at the same time, so it's safe to use the same impl for the return Instr. - InstrIteratorBegin() Instr + InstrIteratorBegin() I // InstrIteratorNext returns the next instruction in this block. Instructions added after lowering must be skipped. // Note: multiple Instr(s) will not be held at the same time, so it's safe to use the same impl for the return Instr. - InstrIteratorNext() Instr + InstrIteratorNext() I // InstrRevIteratorBegin is the same as InstrIteratorBegin, but in the reverse order. - InstrRevIteratorBegin() Instr + InstrRevIteratorBegin() I // InstrRevIteratorNext is the same as InstrIteratorNext, but in the reverse order. - InstrRevIteratorNext() Instr + InstrRevIteratorNext() I // FirstInstr returns the fist instruction in this block where instructions will be inserted after it. - FirstInstr() Instr - // EndInstr returns the end instruction in this block. - EndInstr() Instr + FirstInstr() I // LastInstrForInsertion returns the last instruction in this block where instructions will be inserted before it. // Such insertions only happen when we need to insert spill/reload instructions to adjust the merge edges. // At the time of register allocation, all the critical edges are already split, so there is no need // to worry about the case where branching instruction has multiple successors. // Therefore, usually, it is the nop instruction, but if the block ends with an unconditional branching, then it returns // the unconditional branch, not the nop. In other words it is either nop or unconditional branch. - LastInstrForInsertion() Instr + LastInstrForInsertion() I // Preds returns the number of predecessors of this block in the CFG. Preds() int - // Pred returns the i-th predecessor of this block in the CFG. - Pred(i int) Block // Entry returns true if the block is for the entry block. Entry() bool // Succs returns the number of successors of this block in the CFG. Succs() int - // Succ returns the i-th successor of this block in the CFG. - Succ(i int) Block // LoopHeader returns true if this block is a loop header. LoopHeader() bool // LoopNestingForestChildren returns the number of children of this block in the loop nesting forest. LoopNestingForestChildren() int - // LoopNestingForestChild returns the i-th child of this block in the loop nesting forest. - LoopNestingForestChild(i int) Block } // Instr is an instruction in a block, abstracting away the underlying ISA. Instr interface { + comparable fmt.Stringer - // Next returns the next instruction in the same block. - Next() Instr - // Prev returns the previous instruction in the same block. - Prev() Instr // Defs returns the virtual registers defined by this instruction. Defs(*[]VReg) []VReg // Uses returns the virtual registers used by this instruction. @@ -124,13 +120,5 @@ IsIndirectCall() bool // IsReturn returns true if this instruction is a return instruction. IsReturn() bool - // AddedBeforeRegAlloc returns true if this instruction is added before register allocation. - AddedBeforeRegAlloc() bool - } - - // InstrConstraint is an interface for arch-specific instruction constraints. - InstrConstraint interface { - comparable - Instr } ) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go index eacb6a7ef..a5857f4f2 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go @@ -18,13 +18,13 @@ ) // NewAllocator returns a new Allocator. -func NewAllocator(allocatableRegs *RegisterInfo) Allocator { - a := Allocator{ +func NewAllocator[I Instr, B Block[I], F Function[I, B]](allocatableRegs *RegisterInfo) Allocator[I, B, F] { + a := Allocator[I, B, F]{ regInfo: allocatableRegs, - phiDefInstListPool: wazevoapi.NewPool[phiDefInstList](resetPhiDefInstList), - blockStates: wazevoapi.NewIDedPool[blockState](resetBlockState), + phiDefInstListPool: wazevoapi.NewPool[phiDefInstList[I]](resetPhiDefInstList[I]), + blockStates: wazevoapi.NewIDedPool[blockState[I, B, F]](resetBlockState[I, B, F]), } - a.state.vrStates = wazevoapi.NewIDedPool[vrState](resetVrState) + a.state.vrStates = wazevoapi.NewIDedPool[vrState[I, B, F]](resetVrState[I, B, F]) a.state.reset() for _, regs := range allocatableRegs.AllocatableRegisters { for _, r := range regs { @@ -49,32 +49,39 @@ func NewAllocator(allocatableRegs *RegisterInfo) Allocator { } // Allocator is a register allocator. - Allocator struct { + Allocator[I Instr, B Block[I], F Function[I, B]] struct { // regInfo is static per ABI/ISA, and is initialized by the machine during Machine.PrepareRegisterAllocator. regInfo *RegisterInfo // allocatableSet is a set of allocatable RealReg derived from regInfo. Static per ABI/ISA. allocatableSet RegSet allocatedCalleeSavedRegs []VReg vs []VReg - vs2 []VRegID - phiDefInstListPool wazevoapi.Pool[phiDefInstList] + ss []*vrState[I, B, F] + copies []_copy[I, B, F] + phiDefInstListPool wazevoapi.Pool[phiDefInstList[I]] // Followings are re-used during various places. - blks []Block + blks []B reals []RealReg // Following two fields are updated while iterating the blocks in the reverse postorder. - state state - blockStates wazevoapi.IDedPool[blockState] + state state[I, B, F] + blockStates wazevoapi.IDedPool[blockState[I, B, F]] + } + + // _copy represents a source and destination pair of a copy instruction. + _copy[I Instr, B Block[I], F Function[I, B]] struct { + src *vrState[I, B, F] + dstID VRegID } // programCounter represents an opaque index into the program which is used to represents a LiveInterval of a VReg. programCounter int32 - state struct { + state[I Instr, B Block[I], F Function[I, B]] struct { argRealRegs []VReg - regsInUse regInUseSet - vrStates wazevoapi.IDedPool[vrState] + regsInUse regInUseSet[I, B, F] + vrStates wazevoapi.IDedPool[vrState[I, B, F]] currentBlockID int32 @@ -82,30 +89,30 @@ func NewAllocator(allocatableRegs *RegisterInfo) Allocator { allocatedRegSet RegSet } - blockState struct { + blockState[I Instr, B Block[I], F Function[I, B]] struct { // liveIns is a list of VReg that are live at the beginning of the block. - liveIns []VRegID + liveIns []*vrState[I, B, F] // seen is true if the block is visited during the liveness analysis. seen bool // visited is true if the block is visited during the allocation phase. visited bool startFromPredIndex int // startRegs is a list of RealReg that are used at the beginning of the block. This is used to fix the merge edges. - startRegs regInUseSet + startRegs regInUseSet[I, B, F] // endRegs is a list of RealReg that are used at the end of the block. This is used to fix the merge edges. - endRegs regInUseSet + endRegs regInUseSet[I, B, F] } - vrState struct { + vrState[I Instr, B Block[I], f Function[I, B]] struct { v VReg r RealReg // defInstr is the instruction that defines this value. If this is the phi value and not the entry block, this is nil. - defInstr Instr + defInstr I // defBlk is the block that defines this value. If this is the phi value, this is the block whose arguments contain this value. - defBlk Block + defBlk B // lca = lowest common ancestor. This is the block that is the lowest common ancestor of all the blocks that // reloads this value. This is used to determine the spill location. Only valid if spilled=true. - lca Block + lca B // lastUse is the program counter of the last use of this value. This changes while iterating the block, and // should not be used across the blocks as it becomes invalid. To check the validity, use lastUseUpdatedAtBlockID. lastUse programCounter @@ -120,14 +127,14 @@ func NewAllocator(allocatableRegs *RegisterInfo) Allocator { desiredLoc desiredLoc // phiDefInstList is a list of instructions that defines this phi value. // This is used to determine the spill location, and only valid if isPhi=true. - *phiDefInstList + *phiDefInstList[I] } // phiDefInstList is a linked list of instructions that defines a phi value. - phiDefInstList struct { - instr Instr + phiDefInstList[I Instr] struct { + instr I v VReg - next *phiDefInstList + next *phiDefInstList[I] } // desiredLoc represents a desired location for a VReg. @@ -159,13 +166,14 @@ func (d desiredLoc) stack() bool { return d&3 == desiredLoc(desiredLocKindStack) } -func resetPhiDefInstList(l *phiDefInstList) { - l.instr = nil +func resetPhiDefInstList[I Instr](l *phiDefInstList[I]) { + var nilInstr I + l.instr = nilInstr l.next = nil l.v = VRegInvalid } -func (s *state) dump(info *RegisterInfo) { //nolint:unused +func (s *state[I, B, F]) dump(info *RegisterInfo) { //nolint:unused fmt.Println("\t\tstate:") fmt.Println("\t\t\targRealRegs:", s.argRealRegs) fmt.Println("\t\t\tregsInUse", s.regsInUse.format(info)) @@ -184,7 +192,7 @@ func (s *state) dump(info *RegisterInfo) { //nolint:unused fmt.Println("\t\t\tvrStates:", strings.Join(strs, ", ")) } -func (s *state) reset() { +func (s *state[I, B, F]) reset() { s.argRealRegs = s.argRealRegs[:0] s.vrStates.Reset() s.allocatedRegSet = RegSet(0) @@ -192,79 +200,74 @@ func (s *state) reset() { s.currentBlockID = -1 } -func (s *state) setVRegState(v VReg, r RealReg) { - id := int(v.ID()) - st := s.vrStates.GetOrAllocate(id) - st.r = r - st.v = v -} - -func resetVrState(vs *vrState) { +func resetVrState[I Instr, B Block[I], F Function[I, B]](vs *vrState[I, B, F]) { vs.v = VRegInvalid vs.r = RealRegInvalid - vs.defInstr = nil - vs.defBlk = nil + var nilInstr I + vs.defInstr = nilInstr + var nilBlk B + vs.defBlk = nilBlk vs.spilled = false vs.lastUse = -1 vs.lastUseUpdatedAtBlockID = -1 - vs.lca = nil + vs.lca = nilBlk vs.isPhi = false vs.phiDefInstList = nil vs.desiredLoc = desiredLocUnspecified } -func (s *state) getVRegState(v VRegID) *vrState { - return s.vrStates.GetOrAllocate(int(v)) +func (s *state[I, B, F]) getOrAllocateVRegState(v VReg) *vrState[I, B, F] { + st := s.vrStates.GetOrAllocate(int(v.ID())) + if st.v == VRegInvalid { + st.v = v + } + return st } -func (s *state) useRealReg(r RealReg, v VReg) { - if s.regsInUse.has(r) { - panic("BUG: useRealReg: the given real register is already used") - } - s.regsInUse.add(r, v) - s.setVRegState(v, r) +func (s *state[I, B, F]) getVRegState(v VRegID) *vrState[I, B, F] { + return s.vrStates.Get(int(v)) +} + +func (s *state[I, B, F]) useRealReg(r RealReg, vr *vrState[I, B, F]) { + s.regsInUse.add(r, vr) + vr.r = r s.allocatedRegSet = s.allocatedRegSet.add(r) } -func (s *state) releaseRealReg(r RealReg) { +func (s *state[I, B, F]) releaseRealReg(r RealReg) { current := s.regsInUse.get(r) - if current.Valid() { + if current != nil { s.regsInUse.remove(r) - s.setVRegState(current, RealRegInvalid) + current.r = RealRegInvalid } } // recordReload records that the given VReg is reloaded in the given block. // This is used to determine the spill location by tracking the lowest common ancestor of all the blocks that reloads the value. -func (vs *vrState) recordReload(f Function, blk Block) { +func (vs *vrState[I, B, F]) recordReload(f F, blk B) { vs.spilled = true - if vs.lca == nil { + var nilBlk B + if lca := vs.lca; lca == nilBlk { if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("\t\tv%d is reloaded in blk%d,\n", vs.v.ID(), blk.ID()) } vs.lca = blk - } else { + } else if lca != blk { if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("\t\tv%d is reloaded in blk%d, lca=%d\n", vs.v.ID(), blk.ID(), vs.lca.ID()) } - vs.lca = f.LowestCommonAncestor(vs.lca, blk) + vs.lca = f.LowestCommonAncestor(lca, blk) if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("updated lca=%d\n", vs.lca.ID()) } } } -func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forbiddenMask RegSet, preferred RealReg) (r RealReg) { +func (a *Allocator[I, B, F]) findOrSpillAllocatable(s *state[I, B, F], allocatable []RealReg, forbiddenMask RegSet, preferred RealReg) (r RealReg) { r = RealRegInvalid // First, check if the preferredMask has any allocatable register. if preferred != RealRegInvalid && !forbiddenMask.has(preferred) && !s.regsInUse.has(preferred) { - for _, candidateReal := range allocatable { - // TODO: we should ensure the preferred register is in the allocatable set in the first place, - // but right now, just in case, we check it here. - if candidateReal == preferred { - return preferred - } - } + return preferred } var lastUseAt programCounter @@ -275,7 +278,7 @@ func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forb } using := s.regsInUse.get(candidateReal) - if using == VRegInvalid { + if using == nil { // This is not used at this point. return candidateReal } @@ -284,17 +287,17 @@ func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forb // For example, if the register is used as an argument register, and it might be // spilled and not reloaded when it ends up being used as a temporary to pass // stack based argument. - if using.IsRealReg() { + if using.v.IsRealReg() { continue } isPreferred := candidateReal == preferred // last == -1 means the value won't be used anymore. - if last := s.getVRegState(using.ID()).lastUse; r == RealRegInvalid || isPreferred || last == -1 || (lastUseAt != -1 && last > lastUseAt) { + if last := using.lastUse; r == RealRegInvalid || isPreferred || last == -1 || (lastUseAt != -1 && last > lastUseAt) { lastUseAt = last r = candidateReal - spillVReg = using + spillVReg = using.v if isPreferred { break } @@ -312,7 +315,7 @@ func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forb return r } -func (s *state) findAllocatable(allocatable []RealReg, forbiddenMask RegSet) RealReg { +func (s *state[I, B, F]) findAllocatable(allocatable []RealReg, forbiddenMask RegSet) RealReg { for _, r := range allocatable { if !s.regsInUse.has(r) && !forbiddenMask.has(r) { return r @@ -321,22 +324,20 @@ func (s *state) findAllocatable(allocatable []RealReg, forbiddenMask RegSet) Rea return RealRegInvalid } -func (s *state) resetAt(bs *blockState) { - s.regsInUse.range_(func(_ RealReg, vr VReg) { - s.setVRegState(vr, RealRegInvalid) +func (s *state[I, B, F]) resetAt(bs *blockState[I, B, F]) { + s.regsInUse.range_(func(_ RealReg, vs *vrState[I, B, F]) { + vs.r = RealRegInvalid }) s.regsInUse.reset() - bs.endRegs.range_(func(r RealReg, v VReg) { - id := int(v.ID()) - st := s.vrStates.GetOrAllocate(id) - if st.lastUseUpdatedAtBlockID == s.currentBlockID && st.lastUse == programCounterLiveIn { - s.regsInUse.add(r, v) - s.setVRegState(v, r) + bs.endRegs.range_(func(r RealReg, vs *vrState[I, B, F]) { + if vs.lastUseUpdatedAtBlockID == s.currentBlockID && vs.lastUse == programCounterLiveIn { + s.regsInUse.add(r, vs) + vs.r = r } }) } -func resetBlockState(b *blockState) { +func resetBlockState[I Instr, B Block[I], F Function[I, B]](b *blockState[I, B, F]) { b.seen = false b.visited = false b.endRegs.reset() @@ -345,7 +346,7 @@ func resetBlockState(b *blockState) { b.liveIns = b.liveIns[:0] } -func (b *blockState) dump(a *RegisterInfo) { +func (b *blockState[I, B, F]) dump(a *RegisterInfo) { fmt.Println("\t\tblockState:") fmt.Println("\t\t\tstartRegs:", b.startRegs.format(a)) fmt.Println("\t\t\tendRegs:", b.endRegs.format(a)) @@ -354,13 +355,13 @@ func (b *blockState) dump(a *RegisterInfo) { } // DoAllocation performs register allocation on the given Function. -func (a *Allocator) DoAllocation(f Function) { +func (a *Allocator[I, B, F]) DoAllocation(f F) { a.livenessAnalysis(f) a.alloc(f) a.determineCalleeSavedRealRegs(f) } -func (a *Allocator) determineCalleeSavedRealRegs(f Function) { +func (a *Allocator[I, B, F]) determineCalleeSavedRealRegs(f F) { a.allocatedCalleeSavedRegs = a.allocatedCalleeSavedRegs[:0] a.state.allocatedRegSet.Range(func(allocatedRealReg RealReg) { if a.regInfo.CalleeSavedRegisters.has(allocatedRealReg) { @@ -370,17 +371,17 @@ func (a *Allocator) determineCalleeSavedRealRegs(f Function) { f.ClobberedRegisters(a.allocatedCalleeSavedRegs) } -func (a *Allocator) getOrAllocateBlockState(blockID int32) *blockState { +func (a *Allocator[I, B, F]) getOrAllocateBlockState(blockID int32) *blockState[I, B, F] { return a.blockStates.GetOrAllocate(int(blockID)) } // phiBlk returns the block that defines the given phi value, nil otherwise. -func (s *state) phiBlk(v VRegID) Block { - vs := s.getVRegState(v) +func (vs *vrState[I, B, F]) phiBlk() B { if vs.isPhi { return vs.defBlk } - return nil + var nilBlk B + return nilBlk } const ( @@ -390,31 +391,35 @@ func (s *state) phiBlk(v VRegID) Block { // liveAnalysis constructs Allocator.blockLivenessData. // The algorithm here is described in https://pfalcon.github.io/ssabook/latest/book-full.pdf Chapter 9.2. -func (a *Allocator) livenessAnalysis(f Function) { +func (a *Allocator[I, B, F]) livenessAnalysis(f F) { s := &a.state - for blk := f.PostOrderBlockIteratorBegin(); blk != nil; blk = f.PostOrderBlockIteratorNext() { // Order doesn't matter. + for i := VRegID(0); i < vRegIDReservedForRealNum; i++ { + s.getOrAllocateVRegState(VReg(i).SetRealReg(RealReg(i))) + } + + var nilBlk B + var nilInstr I + for blk := f.PostOrderBlockIteratorBegin(); blk != nilBlk; blk = f.PostOrderBlockIteratorNext() { // We should gather phi value data. - for _, p := range blk.BlockParams(&a.vs) { - vs := s.getVRegState(p.ID()) + for _, p := range f.BlockParams(blk, &a.vs) { + vs := s.getOrAllocateVRegState(p) vs.isPhi = true vs.defBlk = blk } - } - for blk := f.PostOrderBlockIteratorBegin(); blk != nil; blk = f.PostOrderBlockIteratorNext() { blkID := blk.ID() info := a.getOrAllocateBlockState(blkID) - a.vs2 = a.vs2[:0] + a.ss = a.ss[:0] const ( flagDeleted = false flagLive = true ) ns := blk.Succs() for i := 0; i < ns; i++ { - succ := blk.Succ(i) - if succ == nil { + succ := f.Succ(blk, i) + if succ == nilBlk { continue } @@ -424,39 +429,39 @@ func (a *Allocator) livenessAnalysis(f Function) { continue } - for _, v := range succInfo.liveIns { - if s.phiBlk(v) != succ { - st := s.getVRegState(v) + for _, st := range succInfo.liveIns { + if st.phiBlk() != succ && st.spilled != flagLive { //nolint:gosimple // We use .spilled field to store the flag. st.spilled = flagLive - a.vs2 = append(a.vs2, v) + a.ss = append(a.ss, st) } } } - for instr := blk.InstrRevIteratorBegin(); instr != nil; instr = blk.InstrRevIteratorNext() { + for instr := blk.InstrRevIteratorBegin(); instr != nilInstr; instr = blk.InstrRevIteratorNext() { var use, def VReg + var defIsPhi bool for _, def = range instr.Defs(&a.vs) { if !def.IsRealReg() { - id := def.ID() - st := s.getVRegState(id) - // We use .spilled field to store the flag. + st := s.getOrAllocateVRegState(def) + defIsPhi = st.isPhi + // Note: We use .spilled field to store the flag. st.spilled = flagDeleted - a.vs2 = append(a.vs2, id) } } for _, use = range instr.Uses(&a.vs) { if !use.IsRealReg() { - id := use.ID() - st := s.getVRegState(id) - // We use .spilled field to store the flag. - st.spilled = flagLive - a.vs2 = append(a.vs2, id) + st := s.getOrAllocateVRegState(use) + // Note: We use .spilled field to store the flag. + if st.spilled != flagLive { //nolint:gosimple + st.spilled = flagLive + a.ss = append(a.ss, st) + } } } - if def.Valid() && s.phiBlk(def.ID()) != nil { + if defIsPhi { if use.Valid() && use.IsRealReg() { // If the destination is a phi value, and the source is a real register, this is the beginning of the function. a.state.argRealRegs = append(a.state.argRealRegs, use) @@ -464,11 +469,10 @@ func (a *Allocator) livenessAnalysis(f Function) { } } - for _, v := range a.vs2 { - st := s.getVRegState(v) + for _, st := range a.ss { // We use .spilled field to store the flag. if st.spilled == flagLive { //nolint:gosimple - info.liveIns = append(info.liveIns, v) + info.liveIns = append(info.liveIns, st) st.spilled = false } } @@ -479,51 +483,48 @@ func (a *Allocator) livenessAnalysis(f Function) { nrs := f.LoopNestingForestRoots() for i := 0; i < nrs; i++ { root := f.LoopNestingForestRoot(i) - a.loopTreeDFS(root) + a.loopTreeDFS(f, root) } } // loopTreeDFS implements the Algorithm 9.3 in the book in an iterative way. -func (a *Allocator) loopTreeDFS(entry Block) { +func (a *Allocator[I, B, F]) loopTreeDFS(f F, entry B) { a.blks = a.blks[:0] a.blks = append(a.blks, entry) - s := &a.state for len(a.blks) > 0 { tail := len(a.blks) - 1 loop := a.blks[tail] a.blks = a.blks[:tail] - a.vs2 = a.vs2[:0] + a.ss = a.ss[:0] const ( flagDone = false flagPending = true ) info := a.getOrAllocateBlockState(loop.ID()) - for _, v := range info.liveIns { - if s.phiBlk(v) != loop { - a.vs2 = append(a.vs2, v) - st := s.getVRegState(v) + for _, st := range info.liveIns { + if st.phiBlk() != loop { + a.ss = append(a.ss, st) // We use .spilled field to store the flag. st.spilled = flagPending } } - var siblingAddedView []VRegID + var siblingAddedView []*vrState[I, B, F] cn := loop.LoopNestingForestChildren() for i := 0; i < cn; i++ { - child := loop.LoopNestingForestChild(i) + child := f.LoopNestingForestChild(loop, i) childID := child.ID() childInfo := a.getOrAllocateBlockState(childID) if i == 0 { begin := len(childInfo.liveIns) - for _, v := range a.vs2 { - st := s.getVRegState(v) + for _, st := range a.ss { // We use .spilled field to store the flag. if st.spilled == flagPending { //nolint:gosimple st.spilled = flagDone // TODO: deduplicate, though I don't think it has much impact. - childInfo.liveIns = append(childInfo.liveIns, v) + childInfo.liveIns = append(childInfo.liveIns, st) } } siblingAddedView = childInfo.liveIns[begin:] @@ -539,8 +540,7 @@ func (a *Allocator) loopTreeDFS(entry Block) { if cn == 0 { // If there's no forest child, we haven't cleared the .spilled field at this point. - for _, v := range a.vs2 { - st := s.getVRegState(v) + for _, st := range a.ss { st.spilled = false } } @@ -557,37 +557,36 @@ func (a *Allocator) loopTreeDFS(entry Block) { // the spill happens in the block that is the lowest common ancestor of all the blocks that reloads the value. // // All of these logics are almost the same as Go's compiler which has a dedicated description in the source file ^^. -func (a *Allocator) alloc(f Function) { +func (a *Allocator[I, B, F]) alloc(f F) { // First we allocate each block in the reverse postorder (at least one predecessor should be allocated for each block). - for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nil; blk = f.ReversePostOrderBlockIteratorNext() { + var nilBlk B + for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nilBlk; blk = f.ReversePostOrderBlockIteratorNext() { if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("========== allocating blk%d ========\n", blk.ID()) } if blk.Entry() { - a.finalizeStartReg(blk) + a.finalizeStartReg(f, blk) } a.allocBlock(f, blk) } // After the allocation, we all know the start and end state of each block. So we can fix the merge states. - for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nil; blk = f.ReversePostOrderBlockIteratorNext() { + for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nilBlk; blk = f.ReversePostOrderBlockIteratorNext() { a.fixMergeState(f, blk) } // Finally, we insert the spill instructions as we know all the places where the reloads happen. a.scheduleSpills(f) } -func (a *Allocator) updateLiveInVRState(liveness *blockState) { +func (a *Allocator[I, B, F]) updateLiveInVRState(liveness *blockState[I, B, F]) { currentBlockID := a.state.currentBlockID - for _, v := range liveness.liveIns { - vs := a.state.getVRegState(v) + for _, vs := range liveness.liveIns { vs.lastUse = programCounterLiveIn vs.lastUseUpdatedAtBlockID = currentBlockID } } -func (a *Allocator) finalizeStartReg(blk Block) { +func (a *Allocator[I, B, F]) finalizeStartReg(f F, blk B) { bID := blk.ID() - liveness := a.getOrAllocateBlockState(bID) s := &a.state currentBlkState := a.getOrAllocateBlockState(bID) if currentBlkState.startFromPredIndex > -1 { @@ -595,20 +594,20 @@ func (a *Allocator) finalizeStartReg(blk Block) { } s.currentBlockID = bID - a.updateLiveInVRState(liveness) + a.updateLiveInVRState(currentBlkState) preds := blk.Preds() - var predState *blockState + var predState *blockState[I, B, F] switch preds { case 0: // This is the entry block. case 1: - predID := blk.Pred(0).ID() + predID := f.Pred(blk, 0).ID() predState = a.getOrAllocateBlockState(predID) currentBlkState.startFromPredIndex = 0 default: // TODO: there should be some better heuristic to choose the predecessor. for i := 0; i < preds; i++ { - predID := blk.Pred(i).ID() + predID := f.Pred(blk, i).ID() if _predState := a.getOrAllocateBlockState(predID); _predState.visited { predState = _predState currentBlkState.startFromPredIndex = i @@ -621,18 +620,18 @@ func (a *Allocator) finalizeStartReg(blk Block) { panic(fmt.Sprintf("BUG: at lease one predecessor should be visited for blk%d", blk.ID())) } for _, u := range s.argRealRegs { - s.useRealReg(u.RealReg(), u) + s.useRealReg(u.RealReg(), s.getVRegState(u.ID())) } currentBlkState.startFromPredIndex = 0 - } else if predState != nil { + } else { if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("allocating blk%d starting from blk%d (on index=%d) \n", - bID, blk.Pred(currentBlkState.startFromPredIndex).ID(), currentBlkState.startFromPredIndex) + bID, f.Pred(blk, currentBlkState.startFromPredIndex).ID(), currentBlkState.startFromPredIndex) } s.resetAt(predState) } - s.regsInUse.range_(func(allocated RealReg, v VReg) { + s.regsInUse.range_(func(allocated RealReg, v *vrState[I, B, F]) { currentBlkState.startRegs.add(allocated, v) }) if wazevoapi.RegAllocLoggingEnabled { @@ -640,7 +639,7 @@ func (a *Allocator) finalizeStartReg(blk Block) { } } -func (a *Allocator) allocBlock(f Function, blk Block) { +func (a *Allocator[I, B, F]) allocBlock(f F, blk B) { bID := blk.ID() s := &a.state currentBlkState := a.getOrAllocateBlockState(bID) @@ -651,36 +650,34 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } // Clears the previous state. - s.regsInUse.range_(func(allocatedRealReg RealReg, vr VReg) { - s.setVRegState(vr, RealRegInvalid) - }) + s.regsInUse.range_(func(allocatedRealReg RealReg, vr *vrState[I, B, F]) { vr.r = RealRegInvalid }) s.regsInUse.reset() // Then set the start state. - currentBlkState.startRegs.range_(func(allocatedRealReg RealReg, vr VReg) { - s.useRealReg(allocatedRealReg, vr) - }) + currentBlkState.startRegs.range_(func(allocatedRealReg RealReg, vr *vrState[I, B, F]) { s.useRealReg(allocatedRealReg, vr) }) - desiredUpdated := a.vs2[:0] + desiredUpdated := a.ss[:0] // Update the last use of each VReg. + a.copies = a.copies[:0] // Stores the copy instructions. var pc programCounter - for instr := blk.InstrIteratorBegin(); instr != nil; instr = blk.InstrIteratorNext() { - var use, def VReg - for _, use = range instr.Uses(&a.vs) { + var nilInstr I + for instr := blk.InstrIteratorBegin(); instr != nilInstr; instr = blk.InstrIteratorNext() { + var useState *vrState[I, B, F] + for _, use := range instr.Uses(&a.vs) { + useState = s.getVRegState(use.ID()) if !use.IsRealReg() { - s.getVRegState(use.ID()).lastUse = pc + useState.lastUse = pc } } if instr.IsCopy() { - def = instr.Defs(&a.vs)[0] + def := instr.Defs(&a.vs)[0] + a.copies = append(a.copies, _copy[I, B, F]{src: useState, dstID: def.ID()}) r := def.RealReg() if r != RealRegInvalid { - useID := use.ID() - vs := s.getVRegState(useID) - if !vs.isPhi { // TODO: no idea why do we need this. - vs.desiredLoc = newDesiredLocReg(r) - desiredUpdated = append(desiredUpdated, useID) + if !useState.isPhi { // TODO: no idea why do we need this. + useState.desiredLoc = newDesiredLocReg(r) + desiredUpdated = append(desiredUpdated, useState) } } } @@ -689,18 +686,18 @@ func (a *Allocator) allocBlock(f Function, blk Block) { // Mark all live-out values by checking live-in of the successors. // While doing so, we also update the desired register values. - var succ Block + var succ B + var nilBlk B for i, ns := 0, blk.Succs(); i < ns; i++ { - succ = blk.Succ(i) - if succ == nil { + succ = f.Succ(blk, i) + if succ == nilBlk { continue } succID := succ.ID() succState := a.getOrAllocateBlockState(succID) - for _, v := range succState.liveIns { - if s.phiBlk(v) != succ { - st := s.getVRegState(v) + for _, st := range succState.liveIns { + if st.phiBlk() != succ { st.lastUse = programCounterLiveOut } } @@ -709,43 +706,33 @@ func (a *Allocator) allocBlock(f Function, blk Block) { if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("blk%d -> blk%d: start_regs: %s\n", bID, succID, succState.startRegs.format(a.regInfo)) } - succState.startRegs.range_(func(allocatedRealReg RealReg, vr VReg) { - vs := s.getVRegState(vr.ID()) + succState.startRegs.range_(func(allocatedRealReg RealReg, vs *vrState[I, B, F]) { vs.desiredLoc = newDesiredLocReg(allocatedRealReg) - desiredUpdated = append(desiredUpdated, vr.ID()) + desiredUpdated = append(desiredUpdated, vs) }) - for _, p := range succ.BlockParams(&a.vs) { + for _, p := range f.BlockParams(succ, &a.vs) { vs := s.getVRegState(p.ID()) if vs.desiredLoc.realReg() == RealRegInvalid { vs.desiredLoc = desiredLocStack - desiredUpdated = append(desiredUpdated, p.ID()) + desiredUpdated = append(desiredUpdated, vs) } } } } // Propagate the desired register values from the end of the block to the beginning. - for instr := blk.InstrRevIteratorBegin(); instr != nil; instr = blk.InstrRevIteratorNext() { - if instr.IsCopy() { - def := instr.Defs(&a.vs)[0] - defState := s.getVRegState(def.ID()) - desired := defState.desiredLoc.realReg() - if desired == RealRegInvalid { - continue - } - - use := instr.Uses(&a.vs)[0] - useID := use.ID() - useState := s.getVRegState(useID) - if s.phiBlk(useID) != succ && useState.desiredLoc == desiredLocUnspecified { - useState.desiredLoc = newDesiredLocReg(desired) - desiredUpdated = append(desiredUpdated, useID) - } + for _, instr := range a.copies { + defState := s.getVRegState(instr.dstID) + desired := defState.desiredLoc.realReg() + useState := instr.src + if useState.phiBlk() != succ && useState.desiredLoc == desiredLocUnspecified { + useState.desiredLoc = newDesiredLocReg(desired) + desiredUpdated = append(desiredUpdated, useState) } } pc = 0 - for instr := blk.InstrIteratorBegin(); instr != nil; instr = blk.InstrIteratorNext() { + for instr := blk.InstrIteratorBegin(); instr != nilInstr; instr = blk.InstrIteratorNext() { if wazevoapi.RegAllocLoggingEnabled { fmt.Println(instr) } @@ -777,12 +764,12 @@ func (a *Allocator) allocBlock(f Function, blk Block) { r := vs.r if r == RealRegInvalid { - r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[use.RegType()], currentUsedSet, + r = a.findOrSpillAllocatable(s, a.regInfo.AllocatableRegisters[use.RegType()], currentUsedSet, // Prefer the desired register if it's available. vs.desiredLoc.realReg()) vs.recordReload(f, blk) f.ReloadRegisterBefore(use.SetRealReg(r), instr) - s.useRealReg(r, use) + s.useRealReg(r, vs) } if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("\ttrying to use v%v on %s\n", use.ID(), a.regInfo.RealRegName(r)) @@ -799,10 +786,9 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } isIndirect := instr.IsIndirectCall() - call := instr.IsCall() || isIndirect - if call { + if instr.IsCall() || isIndirect { addr := RealRegInvalid - if instr.IsIndirectCall() { + if isIndirect { addr = a.vs[0].RealReg() } a.releaseCallerSavedRegs(addr) @@ -814,8 +800,8 @@ func (a *Allocator) allocBlock(f Function, blk Block) { a.reals = killSet defs := instr.Defs(&a.vs) - switch { - case len(defs) > 1: + switch len(defs) { + default: // Some instructions define multiple values on real registers. // E.g. call instructions (following calling convention) / div instruction on x64 that defines both rax and rdx. // @@ -830,20 +816,21 @@ func (a *Allocator) allocBlock(f Function, blk Block) { if s.regsInUse.has(r) { s.releaseRealReg(r) } - s.useRealReg(r, def) + s.useRealReg(r, s.getVRegState(def.ID())) } - case len(defs) == 1: + case 0: + case 1: def := defs[0] + vState := s.getVRegState(def.ID()) if def.IsRealReg() { r := def.RealReg() if a.allocatableSet.has(r) { if s.regsInUse.has(r) { s.releaseRealReg(r) } - s.useRealReg(r, def) + s.useRealReg(r, vState) } } else { - vState := s.getVRegState(def.ID()) r := vState.r if desired := vState.desiredLoc.realReg(); desired != RealRegInvalid { @@ -864,7 +851,7 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } r = desired s.releaseRealReg(r) - s.useRealReg(r, def) + s.useRealReg(r, vState) } } } @@ -880,9 +867,9 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } if r == RealRegInvalid { typ := def.RegType() - r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[typ], RegSet(0), RealRegInvalid) + r = a.findOrSpillAllocatable(s, a.regInfo.AllocatableRegisters[typ], RegSet(0), RealRegInvalid) } - s.useRealReg(r, def) + s.useRealReg(r, vState) } dr := def.SetRealReg(r) instr.AssignDef(dr) @@ -915,9 +902,7 @@ func (a *Allocator) allocBlock(f Function, blk Block) { pc++ } - s.regsInUse.range_(func(allocated RealReg, v VReg) { - currentBlkState.endRegs.add(allocated, v) - }) + s.regsInUse.range_(func(allocated RealReg, v *vrState[I, B, F]) { currentBlkState.endRegs.add(allocated, v) }) currentBlkState.visited = true if wazevoapi.RegAllocLoggingEnabled { @@ -925,31 +910,30 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } // Reset the desired end location. - for _, v := range desiredUpdated { - vs := s.getVRegState(v) + for _, vs := range desiredUpdated { vs.desiredLoc = desiredLocUnspecified } - a.vs2 = desiredUpdated[:0] + a.ss = desiredUpdated[:0] for i := 0; i < blk.Succs(); i++ { - succ := blk.Succ(i) - if succ == nil { + succ := f.Succ(blk, i) + if succ == nilBlk { continue } // If the successor is not visited yet, finalize the start state. - a.finalizeStartReg(succ) + a.finalizeStartReg(f, succ) } } -func (a *Allocator) releaseCallerSavedRegs(addrReg RealReg) { +func (a *Allocator[I, B, F]) releaseCallerSavedRegs(addrReg RealReg) { s := &a.state for allocated := RealReg(0); allocated < 64; allocated++ { if allocated == addrReg { // If this is the call indirect, we should not touch the addr register. continue } - if v := s.regsInUse.get(allocated); v.Valid() { - if v.IsRealReg() { + if vs := s.regsInUse.get(allocated); vs != nil { + if vs.v.IsRealReg() { continue // This is the argument register as it's already used by VReg backed by the corresponding RealReg. } if !a.regInfo.CallerSavedRegisters.has(allocated) { @@ -961,7 +945,7 @@ func (a *Allocator) releaseCallerSavedRegs(addrReg RealReg) { } } -func (a *Allocator) fixMergeState(f Function, blk Block) { +func (a *Allocator[I, B, F]) fixMergeState(f F, blk B) { preds := blk.Preds() if preds <= 1 { return @@ -975,7 +959,7 @@ func (a *Allocator) fixMergeState(f Function, blk Block) { desiredOccupants := &blkSt.startRegs var desiredOccupantsSet RegSet for i, v := range desiredOccupants { - if v != VRegInvalid { + if v != nil { desiredOccupantsSet = desiredOccupantsSet.add(RealReg(i)) } } @@ -992,7 +976,7 @@ func (a *Allocator) fixMergeState(f Function, blk Block) { continue } - pred := blk.Pred(i) + pred := f.Pred(blk, i) predSt := a.getOrAllocateBlockState(pred.ID()) s.resetAt(predSt) @@ -1012,16 +996,16 @@ func (a *Allocator) fixMergeState(f Function, blk Block) { for r := RealReg(0); r < 64; r++ { desiredVReg := desiredOccupants.get(r) - if !desiredVReg.Valid() { + if desiredVReg == nil { continue } currentVReg := s.regsInUse.get(r) - if desiredVReg.ID() == currentVReg.ID() { + if currentVReg != nil && desiredVReg.v.ID() == currentVReg.v.ID() { continue } - typ := desiredVReg.RegType() + typ := desiredVReg.v.RegType() var tmpRealReg VReg if typ == RegTypeInt { tmpRealReg = intTmp @@ -1039,13 +1023,18 @@ func (a *Allocator) fixMergeState(f Function, blk Block) { // - desiredVReg is the desired VReg value that should be on the register `r`. // - freeReg is the temporary register that can be used to swap the values, which may or may not be used. // - typ is the register type of the `r`. -func (a *Allocator) reconcileEdge(f Function, +func (a *Allocator[I, B, F]) reconcileEdge(f F, r RealReg, - pred Block, - currentVReg, desiredVReg VReg, + pred B, + currentState, desiredState *vrState[I, B, F], freeReg VReg, typ RegType, ) { + desiredVReg := desiredState.v + currentVReg := VRegInvalid + if currentState != nil { + currentVReg = currentState.v + } // There are four cases to consider: // 1. currentVReg is valid, but desiredVReg is on the stack. // 2. Both currentVReg and desiredVReg are valid. @@ -1054,7 +1043,6 @@ func (a *Allocator) reconcileEdge(f Function, s := &a.state if currentVReg.Valid() { - desiredState := s.getVRegState(desiredVReg.ID()) er := desiredState.r if er == RealRegInvalid { // Case 1: currentVReg is valid, but desiredVReg is on the stack. @@ -1068,9 +1056,9 @@ func (a *Allocator) reconcileEdge(f Function, f.StoreRegisterBefore(currentVReg.SetRealReg(r), pred.LastInstrForInsertion()) s.releaseRealReg(r) - s.getVRegState(desiredVReg.ID()).recordReload(f, pred) + desiredState.recordReload(f, pred) f.ReloadRegisterBefore(desiredVReg.SetRealReg(r), pred.LastInstrForInsertion()) - s.useRealReg(r, desiredVReg) + s.useRealReg(r, desiredState) return } else { // Case 2: Both currentVReg and desiredVReg are valid. @@ -1089,8 +1077,8 @@ func (a *Allocator) reconcileEdge(f Function, s.allocatedRegSet = s.allocatedRegSet.add(freeReg.RealReg()) s.releaseRealReg(r) s.releaseRealReg(er) - s.useRealReg(r, desiredVReg) - s.useRealReg(er, currentVReg) + s.useRealReg(r, desiredState) + s.useRealReg(er, currentState) if wazevoapi.RegAllocLoggingEnabled { fmt.Printf("\t\tv%d previously on %s moved to %s\n", currentVReg.ID(), a.regInfo.RealRegName(r), a.regInfo.RealRegName(er)) } @@ -1101,7 +1089,7 @@ func (a *Allocator) reconcileEdge(f Function, desiredVReg.ID(), a.regInfo.RealRegName(r), ) } - if currentReg := s.getVRegState(desiredVReg.ID()).r; currentReg != RealRegInvalid { + if currentReg := desiredState.r; currentReg != RealRegInvalid { // Case 3: Desired is on a different register than `r` and currentReg is not valid. // We simply need to move the desired value to the register. f.InsertMoveBefore( @@ -1113,14 +1101,14 @@ func (a *Allocator) reconcileEdge(f Function, } else { // Case 4: Both currentVReg and desiredVReg are not valid. // We simply need to reload the desired value into the register. - s.getVRegState(desiredVReg.ID()).recordReload(f, pred) + desiredState.recordReload(f, pred) f.ReloadRegisterBefore(desiredVReg.SetRealReg(r), pred.LastInstrForInsertion()) } - s.useRealReg(r, desiredVReg) + s.useRealReg(r, desiredState) } } -func (a *Allocator) scheduleSpills(f Function) { +func (a *Allocator[I, B, F]) scheduleSpills(f F) { states := a.state.vrStates for i := 0; i <= states.MaxIDEncountered(); i++ { vs := states.Get(i) @@ -1133,7 +1121,7 @@ func (a *Allocator) scheduleSpills(f Function) { } } -func (a *Allocator) scheduleSpill(f Function, vs *vrState) { +func (a *Allocator[I, B, F]) scheduleSpill(f F, vs *vrState[I, B, F]) { v := vs.v // If the value is the phi value, we need to insert a spill after each phi definition. if vs.isPhi { @@ -1146,10 +1134,11 @@ func (a *Allocator) scheduleSpill(f Function, vs *vrState) { pos := vs.lca definingBlk := vs.defBlk r := RealRegInvalid - if definingBlk == nil { + var nilBlk B + if definingBlk == nilBlk { panic(fmt.Sprintf("BUG: definingBlk should not be nil for %s. This is likley a bug in backend lowering logic", vs.v.String())) } - if pos == nil { + if pos == nilBlk { panic(fmt.Sprintf("BUG: pos should not be nil for %s. This is likley a bug in backend lowering logic", vs.v.String())) } @@ -1159,7 +1148,7 @@ func (a *Allocator) scheduleSpill(f Function, vs *vrState) { for pos != definingBlk { st := a.getOrAllocateBlockState(pos.ID()) for rr := RealReg(0); rr < 64; rr++ { - if st.startRegs.get(rr) == v { + if vs := st.startRegs.get(rr); vs != nil && vs.v == v { r = rr // Already in the register, so we can place the spill at the beginning of the block. break @@ -1192,7 +1181,7 @@ func (a *Allocator) scheduleSpill(f Function, vs *vrState) { } // Reset resets the allocator's internal state so that it can be reused. -func (a *Allocator) Reset() { +func (a *Allocator[I, B, F]) Reset() { a.state.reset() a.blockStates.Reset() a.phiDefInstListPool.Reset() diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go index 04a8e8f4d..ce84c9c0c 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go @@ -46,52 +46,50 @@ func (rs RegSet) Range(f func(allocatedRealReg RealReg)) { } } -type regInUseSet [64]VReg +type regInUseSet[I Instr, B Block[I], F Function[I, B]] [64]*vrState[I, B, F] -func newRegInUseSet() regInUseSet { - var ret regInUseSet +func newRegInUseSet[I Instr, B Block[I], F Function[I, B]]() regInUseSet[I, B, F] { + var ret regInUseSet[I, B, F] ret.reset() return ret } -func (rs *regInUseSet) reset() { - for i := range rs { - rs[i] = VRegInvalid - } +func (rs *regInUseSet[I, B, F]) reset() { + clear(rs[:]) } -func (rs *regInUseSet) format(info *RegisterInfo) string { //nolint:unused +func (rs *regInUseSet[I, B, F]) format(info *RegisterInfo) string { //nolint:unused var ret []string for i, vr := range rs { - if vr != VRegInvalid { - ret = append(ret, fmt.Sprintf("(%s->v%d)", info.RealRegName(RealReg(i)), vr.ID())) + if vr != nil { + ret = append(ret, fmt.Sprintf("(%s->v%d)", info.RealRegName(RealReg(i)), vr.v.ID())) } } return strings.Join(ret, ", ") } -func (rs *regInUseSet) has(r RealReg) bool { - return r < 64 && rs[r] != VRegInvalid +func (rs *regInUseSet[I, B, F]) has(r RealReg) bool { + return r < 64 && rs[r] != nil } -func (rs *regInUseSet) get(r RealReg) VReg { +func (rs *regInUseSet[I, B, F]) get(r RealReg) *vrState[I, B, F] { return rs[r] } -func (rs *regInUseSet) remove(r RealReg) { - rs[r] = VRegInvalid +func (rs *regInUseSet[I, B, F]) remove(r RealReg) { + rs[r] = nil } -func (rs *regInUseSet) add(r RealReg, vr VReg) { +func (rs *regInUseSet[I, B, F]) add(r RealReg, vr *vrState[I, B, F]) { if r >= 64 { return } rs[r] = vr } -func (rs *regInUseSet) range_(f func(allocatedRealReg RealReg, vr VReg)) { +func (rs *regInUseSet[I, B, F]) range_(f func(allocatedRealReg RealReg, vr *vrState[I, B, F])) { for i, vr := range rs { - if vr != VRegInvalid { + if vr != nil { f(RealReg(i), vr) } } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go index edfa962b5..47a275a3a 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go @@ -1,43 +1,19 @@ package backend import ( - "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" ) // SSAValueDefinition represents a definition of an SSA value. type SSAValueDefinition struct { - // BlockParamValue is valid if Instr == nil - BlockParamValue ssa.Value - - // BlkParamVReg is valid if Instr == nil - BlkParamVReg regalloc.VReg - + V ssa.Value // Instr is not nil if this is a definition from an instruction. Instr *ssa.Instruction - // N is the index of the return value in the instr's return values list. - N int // RefCount is the number of references to the result. - RefCount int + RefCount uint32 } +// IsFromInstr returns true if this definition is from an instruction. func (d *SSAValueDefinition) IsFromInstr() bool { return d.Instr != nil } - -func (d *SSAValueDefinition) IsFromBlockParam() bool { - return d.Instr == nil -} - -func (d *SSAValueDefinition) SSAValue() ssa.Value { - if d.IsFromBlockParam() { - return d.BlockParamValue - } else { - r, rs := d.Instr.Returns() - if d.N == 0 { - return r - } else { - return rs[d.N-1] - } - } -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go index 72ce44e26..639429a63 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go @@ -554,17 +554,21 @@ func (c *callEngine) cloneStack(l uintptr) (newSP, newFP, newTop uintptr, newSta // Copy the existing contents in the previous Go-allocated stack into the new one. var prevStackAligned, newStackAligned []byte { + //nolint:staticcheck sh := (*reflect.SliceHeader)(unsafe.Pointer(&prevStackAligned)) sh.Data = c.stackTop - relSp - setSliceLimits(sh, relSp, relSp) + sh.Len = int(relSp) + sh.Cap = int(relSp) } newTop = alignedStackTop(newStack) { newSP = newTop - relSp newFP = newTop - relFp + //nolint:staticcheck sh := (*reflect.SliceHeader)(unsafe.Pointer(&newStackAligned)) sh.Data = newSP - setSliceLimits(sh, relSp, relSp) + sh.Len = int(relSp) + sh.Cap = int(relSp) } copy(newStackAligned, prevStackAligned) return diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go index 42cc21dcd..eebdba034 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go @@ -275,7 +275,7 @@ func (c *Compiler) LowerToSSA() { builder.DefineVariable(variable, value, entryBlock) c.setWasmLocalVariable(wasm.Index(i), variable) } - c.declareWasmLocals(entryBlock) + c.declareWasmLocals() c.declareNecessaryVariables() c.lowerBody(entryBlock) @@ -295,7 +295,7 @@ func (c *Compiler) setWasmLocalVariable(index wasm.Index, variable ssa.Variable) } // declareWasmLocals declares the SSA variables for the Wasm locals. -func (c *Compiler) declareWasmLocals(entry ssa.BasicBlock) { +func (c *Compiler) declareWasmLocals() { localCount := wasm.Index(len(c.wasmFunctionTyp.Params)) for i, typ := range c.wasmFunctionLocalTypes { st := WasmTypeToSSAType(typ) @@ -543,11 +543,11 @@ func (c *Compiler) initializeCurrentBlockKnownBounds() { cb := &c.bounds[i][c.pointers[i]] if cb.id != smallestID { same = false - break } else { if cb.bound < minBound { minBound = cb.bound } + c.pointers[i]++ } } @@ -555,14 +555,6 @@ func (c *Compiler) initializeCurrentBlockKnownBounds() { // Absolute address cannot be used in the intersection since the value might be only defined in one of the predecessors. c.recordKnownSafeBound(smallestID, minBound, ssa.ValueInvalid) } - - // Move pointer(s) for the smallest ID forward (if same, move all). - for i := 0; i < preds; i++ { - cb := &c.bounds[i][c.pointers[i]] - if cb.id == smallestID { - c.pointers[i]++ - } - } } } } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go index ff963e605..e73debbd1 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go @@ -1538,8 +1538,7 @@ func (c *Compiler) lowerCurrentOpcode() { builder.SetCurrentBlock(elseBlk) case wasm.OpcodeBrTable: - labels := state.tmpForBrTable - labels = labels[:0] + labels := state.tmpForBrTable[:0] labelCount := c.readI32u() for i := 0; i < int(labelCount); i++ { labels = append(labels, c.readI32u()) @@ -1557,6 +1556,7 @@ func (c *Compiler) lowerCurrentOpcode() { } else { c.lowerBrTable(labels, index) } + state.tmpForBrTable = labels // reuse the temporary slice for next use. state.unreachable = true case wasm.OpcodeNop: @@ -4068,13 +4068,14 @@ func (c *Compiler) lowerBrTable(labels []uint32, index ssa.Value) { numArgs = len(f.blockType.Results) } - targets := make([]ssa.BasicBlock, len(labels)) + varPool := builder.VarLengthPool() + trampolineBlockIDs := varPool.Allocate(len(labels)) // We need trampoline blocks since depending on the target block structure, we might end up inserting moves before jumps, // which cannot be done with br_table. Instead, we can do such per-block moves in the trampoline blocks. // At the linking phase (very end of the backend), we can remove the unnecessary jumps, and therefore no runtime overhead. currentBlk := builder.CurrentBlock() - for i, l := range labels { + for _, l := range labels { // Args are always on the top of the stack. Note that we should not share the args slice // among the jump instructions since the args are modified during passes (e.g. redundant phi elimination). args := c.nPeekDup(numArgs) @@ -4082,17 +4083,17 @@ func (c *Compiler) lowerBrTable(labels []uint32, index ssa.Value) { trampoline := builder.AllocateBasicBlock() builder.SetCurrentBlock(trampoline) c.insertJumpToBlock(args, targetBlk) - targets[i] = trampoline + trampolineBlockIDs = trampolineBlockIDs.Append(builder.VarLengthPool(), ssa.Value(trampoline.ID())) } builder.SetCurrentBlock(currentBlk) // If the target block has no arguments, we can just jump to the target block. brTable := builder.AllocateInstruction() - brTable.AsBrTable(index, targets) + brTable.AsBrTable(index, trampolineBlockIDs) builder.InsertInstruction(brTable) - for _, trampoline := range targets { - builder.Seal(trampoline) + for _, trampolineID := range trampolineBlockIDs.View() { + builder.Seal(builder.BasicBlock(ssa.BasicBlockID(trampolineID))) } } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go index 1296706f5..5b055d127 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go @@ -1,5 +1,3 @@ -//go:build go1.21 - package frontend import ( diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go deleted file mode 100644 index 2e786a160..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build !go1.21 - -// TODO: delete after the floor Go version is 1.21 - -package frontend - -import ( - "sort" - - "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" -) - -func sortSSAValueIDs(IDs []ssa.ValueID) { - sort.SliceStable(IDs, func(i, j int) bool { - return int(IDs[i]) < int(IDs[j]) - }) -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go index 8da7347a9..800a5d2a8 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go @@ -16,6 +16,7 @@ func buildHostModuleOpaque(m *wasm.Module, listeners []experimental.FunctionList binary.LittleEndian.PutUint64(ret[0:], uint64(uintptr(unsafe.Pointer(m)))) if len(listeners) > 0 { + //nolint:staticcheck sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&listeners)) binary.LittleEndian.PutUint64(ret[8:], uint64(sliceHeader.Data)) binary.LittleEndian.PutUint64(ret[16:], uint64(sliceHeader.Len)) @@ -33,6 +34,7 @@ func buildHostModuleOpaque(m *wasm.Module, listeners []experimental.FunctionList func hostModuleFromOpaque(opaqueBegin uintptr) *wasm.Module { var opaqueViewOverSlice []byte + //nolint:staticcheck sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverSlice)) sh.Data = opaqueBegin sh.Len = 32 @@ -42,6 +44,7 @@ func hostModuleFromOpaque(opaqueBegin uintptr) *wasm.Module { func hostModuleListenersSliceFromOpaque(opaqueBegin uintptr) []experimental.FunctionListener { var opaqueViewOverSlice []byte + //nolint:staticcheck sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverSlice)) sh.Data = opaqueBegin sh.Len = 32 @@ -51,9 +54,11 @@ func hostModuleListenersSliceFromOpaque(opaqueBegin uintptr) []experimental.Func l := binary.LittleEndian.Uint64(opaqueViewOverSlice[16:]) c := binary.LittleEndian.Uint64(opaqueViewOverSlice[24:]) var ret []experimental.FunctionListener + //nolint:staticcheck sh = (*reflect.SliceHeader)(unsafe.Pointer(&ret)) sh.Data = uintptr(b) - setSliceLimits(sh, uintptr(l), uintptr(c)) + sh.Len = int(l) + sh.Cap = int(c) return ret } @@ -62,6 +67,7 @@ func hostModuleGoFuncFromOpaque[T any](index int, opaqueBegin uintptr) T { ptr := opaqueBegin + offset var opaqueViewOverFunction []byte + //nolint:staticcheck sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverFunction)) sh.Data = ptr sh.Len = 16 diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go deleted file mode 100644 index 6a03fc65c..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !tinygo - -package wazevo - -import "reflect" - -// setSliceLimits sets both Cap and Len for the given reflected slice. -func setSliceLimits(s *reflect.SliceHeader, l, c uintptr) { - s.Len = int(l) - s.Cap = int(c) -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go deleted file mode 100644 index eda3e706a..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build tinygo - -package wazevo - -import "reflect" - -// setSliceLimits sets both Cap and Len for the given reflected slice. -func setSliceLimits(s *reflect.SliceHeader, l, c uintptr) { - s.Len = l - s.Cap = c -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go index 39627b989..cf7f14d3b 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go @@ -34,9 +34,6 @@ type BasicBlock interface { // The returned Value is the definition of the param in this block. Param(i int) Value - // InsertInstruction inserts an instruction that implements Value into the tail of this block. - InsertInstruction(raw *Instruction) - // Root returns the root instruction of this block. Root() *Instruction @@ -81,7 +78,7 @@ type BasicBlock interface { rootInstr, currentInstr *Instruction // params are Values that represent parameters to a basicBlock. // Each parameter can be considered as an output of PHI instruction in traditional SSA. - params []Value + params Values preds []basicBlockPredecessorInfo success []*basicBlock // singlePred is the alias to preds[0] for fast lookup, and only set after Seal is called. @@ -179,23 +176,23 @@ func (bb *basicBlock) ReturnBlock() bool { // AddParam implements BasicBlock.AddParam. func (bb *basicBlock) AddParam(b Builder, typ Type) Value { paramValue := b.allocateValue(typ) - bb.params = append(bb.params, paramValue) + bb.params = bb.params.Append(&b.(*builder).varLengthPool, paramValue) return paramValue } // addParamOn adds a parameter to this block whose value is already allocated. -func (bb *basicBlock) addParamOn(value Value) { - bb.params = append(bb.params, value) +func (bb *basicBlock) addParamOn(b *builder, value Value) { + bb.params = bb.params.Append(&b.varLengthPool, value) } // Params implements BasicBlock.Params. func (bb *basicBlock) Params() int { - return len(bb.params) + return len(bb.params.View()) } // Param implements BasicBlock.Param. func (bb *basicBlock) Param(i int) Value { - return bb.params[i] + return bb.params.View()[i] } // Valid implements BasicBlock.Valid. @@ -208,8 +205,8 @@ func (bb *basicBlock) Sealed() bool { return bb.sealed } -// InsertInstruction implements BasicBlock.InsertInstruction. -func (bb *basicBlock) InsertInstruction(next *Instruction) { +// insertInstruction implements BasicBlock.InsertInstruction. +func (bb *basicBlock) insertInstruction(b *builder, next *Instruction) { current := bb.currentInstr if current != nil { current.next = next @@ -221,12 +218,12 @@ func (bb *basicBlock) InsertInstruction(next *Instruction) { switch next.opcode { case OpcodeJump, OpcodeBrz, OpcodeBrnz: - target := next.blk.(*basicBlock) - target.addPred(bb, next) + target := BasicBlockID(next.rValue) + b.basicBlock(target).addPred(bb, next) case OpcodeBrTable: - for _, _target := range next.targets { - target := _target.(*basicBlock) - target.addPred(bb, next) + for _, _target := range next.rValues.View() { + target := BasicBlockID(_target) + b.basicBlock(target).addPred(bb, next) } } } @@ -268,7 +265,7 @@ func (bb *basicBlock) Tail() *Instruction { // reset resets the basicBlock to its initial state so that it can be reused for another function. func resetBasicBlock(bb *basicBlock) { - bb.params = bb.params[:0] + bb.params = ValuesNil bb.rootInstr, bb.currentInstr = nil, nil bb.preds = bb.preds[:0] bb.success = bb.success[:0] @@ -310,8 +307,8 @@ func (bb *basicBlock) addPred(blk BasicBlock, branch *Instruction) { // formatHeader returns the string representation of the header of the basicBlock. func (bb *basicBlock) formatHeader(b Builder) string { - ps := make([]string, len(bb.params)) - for i, p := range bb.params { + ps := make([]string, len(bb.params.View())) + for i, p := range bb.params.View() { ps[i] = p.formatWithType(b) } @@ -339,7 +336,9 @@ func (bb *basicBlock) validate(b *builder) { if len(bb.preds) > 0 { for _, pred := range bb.preds { if pred.branch.opcode != OpcodeBrTable { - if target := pred.branch.blk; target != bb { + blockID := int(pred.branch.rValue) + target := b.basicBlocksPool.View(blockID) + if target != bb { panic(fmt.Sprintf("BUG: '%s' is not branch to %s, but to %s", pred.branch.Format(b), bb.Name(), target.Name())) } @@ -349,14 +348,14 @@ func (bb *basicBlock) validate(b *builder) { if bb.ReturnBlock() { exp = len(b.currentSignature.Results) } else { - exp = len(bb.params) + exp = len(bb.params.View()) } if len(pred.branch.vs.View()) != exp { panic(fmt.Sprintf( "BUG: len(argument at %s) != len(params at %s): %d != %d: %s", pred.blk.Name(), bb.Name(), - len(pred.branch.vs.View()), len(bb.params), pred.branch.Format(b), + len(pred.branch.vs.View()), len(bb.params.View()), pred.branch.Format(b), )) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go index e1471edc3..fb98298f7 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go @@ -1,5 +1,3 @@ -//go:build go1.21 - package ssa import ( diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go deleted file mode 100644 index 9dc881dae..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !go1.21 - -// TODO: delete after the floor Go version is 1.21 - -package ssa - -import "sort" - -func sortBlocks(blocks []*basicBlock) { - sort.SliceStable(blocks, func(i, j int) bool { - iBlk, jBlk := blocks[i], blocks[j] - if jBlk.ReturnBlock() { - return true - } - if iBlk.ReturnBlock() { - return false - } - iRoot, jRoot := iBlk.rootInstr, jBlk.rootInstr - if iRoot == nil || jRoot == nil { // For testing. - return true - } - return iBlk.rootInstr.id < jBlk.rootInstr.id - }) -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go index 0b700c4b1..43dd7d292 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go @@ -94,9 +94,9 @@ type Builder interface { // Returns nil if there's no unseen BasicBlock. BlockIteratorNext() BasicBlock - // ValueRefCounts returns the map of ValueID to its reference count. - // The returned slice must not be modified. - ValueRefCounts() []int + // ValuesInfo returns the data per Value used to lower the SSA in backend. + // This is indexed by ValueID. + ValuesInfo() []ValueInfo // BlockIteratorReversePostOrderBegin is almost the same as BlockIteratorBegin except it returns the BasicBlock in the reverse post-order. // This is available after RunPasses is run. @@ -129,20 +129,24 @@ type Builder interface { // InsertZeroValue inserts a zero value constant instruction of the given type. InsertZeroValue(t Type) + + // BasicBlock returns the BasicBlock of the given ID. + BasicBlock(id BasicBlockID) BasicBlock + + // InstructionOfValue returns the Instruction that produces the given Value or nil if the Value is not produced by any Instruction. + InstructionOfValue(v Value) *Instruction } // NewBuilder returns a new Builder implementation. func NewBuilder() Builder { return &builder{ - instructionsPool: wazevoapi.NewPool[Instruction](resetInstruction), - basicBlocksPool: wazevoapi.NewPool[basicBlock](resetBasicBlock), - varLengthBasicBlockPool: wazevoapi.NewVarLengthPool[BasicBlock](), - varLengthPool: wazevoapi.NewVarLengthPool[Value](), - valueAnnotations: make(map[ValueID]string), - signatures: make(map[SignatureID]*Signature), - valueIDAliases: make(map[ValueID]Value), - redundantParameterIndexToValue: make(map[int]Value), - returnBlk: &basicBlock{id: basicBlockIDReturnBlock}, + instructionsPool: wazevoapi.NewPool[Instruction](resetInstruction), + basicBlocksPool: wazevoapi.NewPool[basicBlock](resetBasicBlock), + varLengthBasicBlockPool: wazevoapi.NewVarLengthPool[BasicBlock](), + varLengthPool: wazevoapi.NewVarLengthPool[Value](), + valueAnnotations: make(map[ValueID]string), + signatures: make(map[SignatureID]*Signature), + returnBlk: &basicBlock{id: basicBlockIDReturnBlock}, } } @@ -159,19 +163,16 @@ type builder struct { currentBB *basicBlock returnBlk *basicBlock - // variables track the types for Variable with the index regarded Variable. - variables []Type // nextValueID is used by builder.AllocateValue. nextValueID ValueID // nextVariable is used by builder.AllocateVariable. nextVariable Variable - valueIDAliases map[ValueID]Value + // valueAnnotations contains the annotations for each Value, only used for debugging. valueAnnotations map[ValueID]string - // valueRefCounts is used to lower the SSA in backend, and will be calculated - // by the last SSA-level optimization pass. - valueRefCounts []int + // valuesInfo contains the data per Value used to lower the SSA in backend. This is indexed by ValueID. + valuesInfo []ValueInfo // dominators stores the immediate dominator of each BasicBlock. // The index is blockID of the BasicBlock. @@ -184,12 +185,10 @@ type builder struct { loopNestingForestRoots []BasicBlock // The followings are used for optimization passes/deterministic compilation. - instStack []*Instruction - valueIDToInstruction []*Instruction - blkStack []*basicBlock - blkStack2 []*basicBlock - ints []int - redundantParameterIndexToValue map[int]Value + instStack []*Instruction + blkStack []*basicBlock + blkStack2 []*basicBlock + redundantParams []redundantParam // blockIterCur is used to implement blockIteratorBegin and blockIteratorNext. blockIterCur int @@ -207,6 +206,34 @@ type builder struct { zeros [typeEnd]Value } +// ValueInfo contains the data per Value used to lower the SSA in backend. +type ValueInfo struct { + // RefCount is the reference count of the Value. + RefCount uint32 + alias Value +} + +// redundantParam is a pair of the index of the redundant parameter and the Value. +// This is used to eliminate the redundant parameters in the optimization pass. +type redundantParam struct { + // index is the index of the redundant parameter in the basicBlock. + index int + // uniqueValue is the Value which is passed to the redundant parameter. + uniqueValue Value +} + +// BasicBlock implements Builder.BasicBlock. +func (b *builder) BasicBlock(id BasicBlockID) BasicBlock { + return b.basicBlock(id) +} + +func (b *builder) basicBlock(id BasicBlockID) *basicBlock { + if id == basicBlockIDReturnBlock { + return b.returnBlk + } + return b.basicBlocksPool.View(int(id)) +} + // InsertZeroValue implements Builder.InsertZeroValue. func (b *builder) InsertZeroValue(t Type) { if b.zeros[t].Valid() { @@ -256,7 +283,7 @@ func (b *builder) Init(s *Signature) { sig.used = false } - b.ints = b.ints[:0] + b.redundantParams = b.redundantParams[:0] b.blkStack = b.blkStack[:0] b.blkStack2 = b.blkStack2[:0] b.dominators = b.dominators[:0] @@ -265,17 +292,11 @@ func (b *builder) Init(s *Signature) { for v := ValueID(0); v < b.nextValueID; v++ { delete(b.valueAnnotations, v) - delete(b.valueIDAliases, v) - b.valueRefCounts[v] = 0 - b.valueIDToInstruction[v] = nil + b.valuesInfo[v] = ValueInfo{alias: ValueInvalid} } b.nextValueID = 0 b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0] b.doneBlockLayout = false - for i := range b.valueRefCounts { - b.valueRefCounts[i] = 0 - } - b.currentSourceOffset = sourceOffsetUnknown } @@ -355,7 +376,7 @@ func (b *builder) Idom(blk BasicBlock) BasicBlock { // InsertInstruction implements Builder.InsertInstruction. func (b *builder) InsertInstruction(instr *Instruction) { - b.currentBB.InsertInstruction(instr) + b.currentBB.insertInstruction(b, instr) if l := b.currentSourceOffset; l.Valid() { // Emit the source offset info only when the instruction has side effect because @@ -377,7 +398,7 @@ func (b *builder) InsertInstruction(instr *Instruction) { } r1 := b.allocateValue(t1) - instr.rValue = r1 + instr.rValue = r1.setInstructionID(instr.id) tsl := len(ts) if tsl == 0 { @@ -386,20 +407,14 @@ func (b *builder) InsertInstruction(instr *Instruction) { rValues := b.varLengthPool.Allocate(tsl) for i := 0; i < tsl; i++ { - rValues = rValues.Append(&b.varLengthPool, b.allocateValue(ts[i])) + rn := b.allocateValue(ts[i]) + rValues = rValues.Append(&b.varLengthPool, rn.setInstructionID(instr.id)) } instr.rValues = rValues } // DefineVariable implements Builder.DefineVariable. func (b *builder) DefineVariable(variable Variable, value Value, block BasicBlock) { - if b.variables[variable].invalid() { - panic("BUG: trying to define variable " + variable.String() + " but is not declared yet") - } - - if b.variables[variable] != value.Type() { - panic(fmt.Sprintf("BUG: inconsistent type for variable %d: expected %s but got %s", variable, b.variables[variable], value.Type())) - } bb := block.(*basicBlock) bb.lastDefinitions[variable] = value } @@ -426,20 +441,9 @@ func (b *builder) EntryBlock() BasicBlock { // DeclareVariable implements Builder.DeclareVariable. func (b *builder) DeclareVariable(typ Type) Variable { - v := b.allocateVariable() - iv := int(v) - if l := len(b.variables); l <= iv { - b.variables = append(b.variables, make([]Type, 2*(l+1))...) - } - b.variables[v] = typ - return v -} - -// allocateVariable allocates a new variable. -func (b *builder) allocateVariable() (ret Variable) { - ret = b.nextVariable + v := b.nextVariable b.nextVariable++ - return + return v.setType(typ) } // allocateValue implements Builder.AllocateValue. @@ -475,8 +479,7 @@ func (b *builder) findValueInLinearPath(variable Variable, blk *basicBlock) Valu // MustFindValue implements Builder.MustFindValue. func (b *builder) MustFindValue(variable Variable) Value { - typ := b.definedVariableType(variable) - return b.findValue(typ, variable, b.currentBB) + return b.findValue(variable.getType(), variable, b.currentBB) } // findValue recursively tries to find the latest definition of a `variable`. The algorithm is described in @@ -504,7 +507,7 @@ func (b *builder) findValue(typ Type, variable Variable, blk *basicBlock) Value return value } else if blk.EntryBlock() { // If this is the entry block, we reach the uninitialized variable which has zero value. - return b.zeros[b.definedVariableType(variable)] + return b.zeros[variable.getType()] } if pred := blk.singlePred; pred != nil { @@ -536,14 +539,13 @@ func (b *builder) findValue(typ Type, variable Variable, blk *basicBlock) Value if uniqueValue != ValueInvalid { // If all the predecessors have the same definition, we can use that value. - b.DefineVariable(variable, uniqueValue, blk) b.alias(tmpValue, uniqueValue) return uniqueValue } else { // Otherwise, add the tmpValue to this block as a parameter which may or may not be redundant, but // later we eliminate trivial params in an optimization pass. This must be done before finding the // definitions in the predecessors so that we can break the cycle. - blk.addParamOn(tmpValue) + blk.addParamOn(b, tmpValue) // After the new param is added, we have to manipulate the original branching instructions // in predecessors so that they would pass the definition of `variable` as the argument to // the newly added PHI. @@ -566,8 +568,8 @@ func (b *builder) Seal(raw BasicBlock) { for _, v := range blk.unknownValues { variable, phiValue := v.variable, v.value - typ := b.definedVariableType(variable) - blk.addParamOn(phiValue) + typ := variable.getType() + blk.addParamOn(b, phiValue) for i := range blk.preds { pred := &blk.preds[i] predValue := b.findValue(typ, variable, pred.blk) @@ -579,15 +581,6 @@ func (b *builder) Seal(raw BasicBlock) { } } -// definedVariableType returns the type of the given variable. If the variable is not defined yet, it panics. -func (b *builder) definedVariableType(variable Variable) Type { - typ := b.variables[variable] - if typ.invalid() { - panic(fmt.Sprintf("%s is not defined yet", variable)) - } - return typ -} - // Format implements Builder.Format. func (b *builder) Format() string { str := strings.Builder{} @@ -689,15 +682,24 @@ func (b *builder) blockIteratorReversePostOrderNext() *basicBlock { } } -// ValueRefCounts implements Builder.ValueRefCounts. -func (b *builder) ValueRefCounts() []int { - return b.valueRefCounts +// ValuesInfo implements Builder.ValuesInfo. +func (b *builder) ValuesInfo() []ValueInfo { + return b.valuesInfo } // alias records the alias of the given values. The alias(es) will be // eliminated in the optimization pass via resolveArgumentAlias. func (b *builder) alias(dst, src Value) { - b.valueIDAliases[dst.ID()] = src + did := int(dst.ID()) + if did >= len(b.valuesInfo) { + l := did + 1 - len(b.valuesInfo) + b.valuesInfo = append(b.valuesInfo, make([]ValueInfo, l)...) + view := b.valuesInfo[len(b.valuesInfo)-l:] + for i := range view { + view[i].alias = ValueInvalid + } + } + b.valuesInfo[did].alias = src } // resolveArgumentAlias resolves the alias of the arguments of the given instruction. @@ -722,10 +724,13 @@ func (b *builder) resolveArgumentAlias(instr *Instruction) { // resolveAlias resolves the alias of the given value. func (b *builder) resolveAlias(v Value) Value { + info := b.valuesInfo + l := ValueID(len(info)) // Some aliases are chained, so we need to resolve them recursively. for { - if src, ok := b.valueIDAliases[v.ID()]; ok { - v = src + vid := v.ID() + if vid < l && info[vid].alias.Valid() { + v = info[vid].alias } else { break } @@ -773,3 +778,13 @@ func (b *builder) LoopNestingForestRoots() []BasicBlock { func (b *builder) LowestCommonAncestor(blk1, blk2 BasicBlock) BasicBlock { return b.sparseTree.findLCA(blk1.ID(), blk2.ID()) } + +// InstructionOfValue returns the instruction that produces the given Value, or nil +// if the Value is not produced by any instruction. +func (b *builder) InstructionOfValue(v Value) *Instruction { + instrID := v.instructionID() + if instrID <= 0 { + return nil + } + return b.instructionsPool.View(instrID - 1) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go index 3e3482efc..9a3d1da6e 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go @@ -25,11 +25,13 @@ type Instruction struct { v3 Value vs Values typ Type - blk BasicBlock - targets []BasicBlock prev, next *Instruction - rValue Value + // rValue is the (first) return value of this instruction. + // For branching instructions except for OpcodeBrTable, they hold BlockID to jump cast to Value. + rValue Value + // rValues are the rest of the return values of this instruction. + // For OpcodeBrTable, it holds the list of BlockID to jump cast to Value. rValues Values gid InstructionGroupID sourceOffset SourceOffset @@ -105,6 +107,9 @@ type Instruct // Returns Value(s) produced by this instruction if any. // The `first` is the first return value, and `rest` is the rest of the values. func (i *Instruction) Returns() (first Value, rest []Value) { + if i.IsBranching() { + return ValueInvalid, nil + } return i.rValue, i.rValues.View() } @@ -2077,7 +2082,7 @@ func (i *Instruction) InvertBrx() { } // BranchData returns the branch data for this instruction necessary for backends. -func (i *Instruction) BranchData() (condVal Value, blockArgs []Value, target BasicBlock) { +func (i *Instruction) BranchData() (condVal Value, blockArgs []Value, target BasicBlockID) { switch i.opcode { case OpcodeJump: condVal = ValueInvalid @@ -2087,17 +2092,17 @@ func (i *Instruction) BranchData() (condVal Value, blockArgs []Value, target Bas panic("BUG") } blockArgs = i.vs.View() - target = i.blk + target = BasicBlockID(i.rValue) return } // BrTableData returns the branch table data for this instruction necessary for backends. -func (i *Instruction) BrTableData() (index Value, targets []BasicBlock) { +func (i *Instruction) BrTableData() (index Value, targets Values) { if i.opcode != OpcodeBrTable { panic("BUG: BrTableData only available for OpcodeBrTable") } index = i.v - targets = i.targets + targets = i.rValues return } @@ -2105,7 +2110,7 @@ func (i *Instruction) BrTableData() (index Value, targets []BasicBlock) { func (i *Instruction) AsJump(vs Values, target BasicBlock) *Instruction { i.opcode = OpcodeJump i.vs = vs - i.blk = target + i.rValue = Value(target.ID()) return i } @@ -2130,7 +2135,7 @@ func (i *Instruction) AsBrz(v Value, args Values, target BasicBlock) { i.opcode = OpcodeBrz i.v = v i.vs = args - i.blk = target + i.rValue = Value(target.ID()) } // AsBrnz initializes this instruction as a branch-if-not-zero instruction with OpcodeBrnz. @@ -2138,15 +2143,16 @@ func (i *Instruction) AsBrnz(v Value, args Values, target BasicBlock) *Instructi i.opcode = OpcodeBrnz i.v = v i.vs = args - i.blk = target + i.rValue = Value(target.ID()) return i } // AsBrTable initializes this instruction as a branch-table instruction with OpcodeBrTable. -func (i *Instruction) AsBrTable(index Value, targets []BasicBlock) { +// targets is a list of basic block IDs cast to Values. +func (i *Instruction) AsBrTable(index Value, targets Values) { i.opcode = OpcodeBrTable i.v = index - i.targets = targets + i.rValues = targets } // AsCall initializes this instruction as a call instruction with OpcodeCall. @@ -2531,7 +2537,8 @@ func (i *Instruction) Format(b Builder) string { if i.IsFallthroughJump() { vs[0] = " fallthrough" } else { - vs[0] = " " + i.blk.(*basicBlock).Name() + blockId := BasicBlockID(i.rValue) + vs[0] = " " + b.BasicBlock(blockId).Name() } for idx := range view { vs[idx+1] = view[idx].Format(b) @@ -2542,7 +2549,8 @@ func (i *Instruction) Format(b Builder) string { view := i.vs.View() vs := make([]string, len(view)+2) vs[0] = " " + i.v.Format(b) - vs[1] = i.blk.(*basicBlock).Name() + blockId := BasicBlockID(i.rValue) + vs[1] = b.BasicBlock(blockId).Name() for idx := range view { vs[idx+2] = view[idx].Format(b) } @@ -2551,8 +2559,8 @@ func (i *Instruction) Format(b Builder) string { // `BrTable index, [label1, label2, ... labelN]` instSuffix = fmt.Sprintf(" %s", i.v.Format(b)) instSuffix += ", [" - for i, target := range i.targets { - blk := target.(*basicBlock) + for i, target := range i.rValues.View() { + blk := b.BasicBlock(BasicBlockID(target)) if i == 0 { instSuffix += blk.Name() } else { @@ -2621,11 +2629,12 @@ func (i *Instruction) Format(b Builder) string { instr := i.opcode.String() + instSuffix var rvs []string - if rv := i.rValue; rv.Valid() { - rvs = append(rvs, rv.formatWithType(b)) + r1, rs := i.Returns() + if r1.Valid() { + rvs = append(rvs, r1.formatWithType(b)) } - for _, v := range i.rValues.View() { + for _, v := range rs { rvs = append(rvs, v.formatWithType(b)) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go index 89ec34b7e..b9763791d 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go @@ -112,7 +112,7 @@ func passDeadBlockEliminationOpt(b *builder) { // This requires the reverse post-order traversal to be calculated before calling this function, // hence passCalculateImmediateDominators must be called before this. func passRedundantPhiEliminationOpt(b *builder) { - redundantParameterIndexes := b.ints[:0] // reuse the slice from previous iterations. + redundantParams := b.redundantParams[:0] // reuse the slice from previous iterations. // TODO: this might be costly for large programs, but at least, as far as I did the experiment, it's almost the // same as the single iteration version in terms of the overall compilation time. That *might be* mostly thanks to the fact @@ -128,10 +128,11 @@ func passRedundantPhiEliminationOpt(b *builder) { _ = b.blockIteratorReversePostOrderBegin() // skip entry block! // Below, we intentionally use the named iteration variable name, as this comes with inevitable nested for loops! for blk := b.blockIteratorReversePostOrderNext(); blk != nil; blk = b.blockIteratorReversePostOrderNext() { - paramNum := len(blk.params) + params := blk.params.View() + paramNum := len(params) for paramIndex := 0; paramIndex < paramNum; paramIndex++ { - phiValue := blk.params[paramIndex] + phiValue := params[paramIndex] redundant := true nonSelfReferencingValue := ValueInvalid @@ -162,55 +163,58 @@ func passRedundantPhiEliminationOpt(b *builder) { } if redundant { - b.redundantParameterIndexToValue[paramIndex] = nonSelfReferencingValue - redundantParameterIndexes = append(redundantParameterIndexes, paramIndex) + redundantParams = append(redundantParams, redundantParam{ + index: paramIndex, uniqueValue: nonSelfReferencingValue, + }) } } - if len(b.redundantParameterIndexToValue) == 0 { + if len(redundantParams) == 0 { continue } changed = true // Remove the redundant PHIs from the argument list of branching instructions. for predIndex := range blk.preds { - var cur int + redundantParamsCur, predParamCur := 0, 0 predBlk := blk.preds[predIndex] branchInst := predBlk.branch view := branchInst.vs.View() for argIndex, value := range view { - if _, ok := b.redundantParameterIndexToValue[argIndex]; !ok { - view[cur] = value - cur++ + if len(redundantParams) == redundantParamsCur || + redundantParams[redundantParamsCur].index != argIndex { + view[predParamCur] = value + predParamCur++ + } else { + redundantParamsCur++ } } - branchInst.vs.Cut(cur) + branchInst.vs.Cut(predParamCur) } // Still need to have the definition of the value of the PHI (previously as the parameter). - for _, redundantParamIndex := range redundantParameterIndexes { - phiValue := blk.params[redundantParamIndex] - onlyValue := b.redundantParameterIndexToValue[redundantParamIndex] + for i := range redundantParams { + redundantValue := &redundantParams[i] + phiValue := params[redundantValue.index] // Create an alias in this block from the only phi argument to the phi value. - b.alias(phiValue, onlyValue) + b.alias(phiValue, redundantValue.uniqueValue) } // Finally, Remove the param from the blk. - var cur int + paramsCur, redundantParamsCur := 0, 0 for paramIndex := 0; paramIndex < paramNum; paramIndex++ { - param := blk.params[paramIndex] - if _, ok := b.redundantParameterIndexToValue[paramIndex]; !ok { - blk.params[cur] = param - cur++ + param := params[paramIndex] + if len(redundantParams) == redundantParamsCur || redundantParams[redundantParamsCur].index != paramIndex { + params[paramsCur] = param + paramsCur++ + } else { + redundantParamsCur++ } } - blk.params = blk.params[:cur] + blk.params.Cut(paramsCur) // Clears the map for the next iteration. - for _, paramIndex := range redundantParameterIndexes { - delete(b.redundantParameterIndexToValue, paramIndex) - } - redundantParameterIndexes = redundantParameterIndexes[:0] + redundantParams = redundantParams[:0] } if !changed { @@ -219,7 +223,7 @@ func passRedundantPhiEliminationOpt(b *builder) { } // Reuse the slice for the future passes. - b.ints = redundantParameterIndexes + b.redundantParams = redundantParams } // passDeadCodeEliminationOpt traverses all the instructions, and calculates the reference count of each Value, and @@ -231,11 +235,13 @@ func passRedundantPhiEliminationOpt(b *builder) { // TODO: the algorithm here might not be efficient. Get back to this later. func passDeadCodeEliminationOpt(b *builder) { nvid := int(b.nextValueID) - if nvid >= len(b.valueRefCounts) { - b.valueRefCounts = append(b.valueRefCounts, make([]int, nvid-len(b.valueRefCounts)+1)...) - } - if nvid >= len(b.valueIDToInstruction) { - b.valueIDToInstruction = append(b.valueIDToInstruction, make([]*Instruction, nvid-len(b.valueIDToInstruction)+1)...) + if nvid >= len(b.valuesInfo) { + l := nvid - len(b.valuesInfo) + 1 + b.valuesInfo = append(b.valuesInfo, make([]ValueInfo, l)...) + view := b.valuesInfo[len(b.valuesInfo)-l:] + for i := range view { + view[i].alias = ValueInvalid + } } // First, we gather all the instructions with side effects. @@ -255,14 +261,6 @@ func passDeadCodeEliminationOpt(b *builder) { // The strict side effect should create different instruction groups. gid++ } - - r1, rs := cur.Returns() - if r1.Valid() { - b.valueIDToInstruction[r1.ID()] = cur - } - for _, r := range rs { - b.valueIDToInstruction[r.ID()] = cur - } } } @@ -283,28 +281,28 @@ func passDeadCodeEliminationOpt(b *builder) { v1, v2, v3, vs := live.Args() if v1.Valid() { - producingInst := b.valueIDToInstruction[v1.ID()] + producingInst := b.InstructionOfValue(v1) if producingInst != nil { liveInstructions = append(liveInstructions, producingInst) } } if v2.Valid() { - producingInst := b.valueIDToInstruction[v2.ID()] + producingInst := b.InstructionOfValue(v2) if producingInst != nil { liveInstructions = append(liveInstructions, producingInst) } } if v3.Valid() { - producingInst := b.valueIDToInstruction[v3.ID()] + producingInst := b.InstructionOfValue(v3) if producingInst != nil { liveInstructions = append(liveInstructions, producingInst) } } for _, v := range vs { - producingInst := b.valueIDToInstruction[v.ID()] + producingInst := b.InstructionOfValue(v) if producingInst != nil { liveInstructions = append(liveInstructions, producingInst) } @@ -352,34 +350,19 @@ func (b *builder) incRefCount(id ValueID, from *Instruction) { if wazevoapi.SSALoggingEnabled { fmt.Printf("v%d referenced from %v\n", id, from.Format(b)) } - b.valueRefCounts[id]++ + info := &b.valuesInfo[id] + info.RefCount++ } // passNopInstElimination eliminates the instructions which is essentially a no-op. func passNopInstElimination(b *builder) { - if int(b.nextValueID) >= len(b.valueIDToInstruction) { - b.valueIDToInstruction = append(b.valueIDToInstruction, make([]*Instruction, int(b.nextValueID)-len(b.valueIDToInstruction)+1)...) - } - - for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { - for cur := blk.rootInstr; cur != nil; cur = cur.next { - r1, rs := cur.Returns() - if r1.Valid() { - b.valueIDToInstruction[r1.ID()] = cur - } - for _, r := range rs { - b.valueIDToInstruction[r.ID()] = cur - } - } - } - for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { for cur := blk.rootInstr; cur != nil; cur = cur.next { switch cur.Opcode() { // TODO: add more logics here. case OpcodeIshl, OpcodeSshr, OpcodeUshr: x, amount := cur.Arg2() - definingInst := b.valueIDToInstruction[amount.ID()] + definingInst := b.InstructionOfValue(amount) if definingInst == nil { // If there's no defining instruction, that means the amount is coming from the parameter. continue diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go index 584b5eade..0118e8b2e 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go @@ -33,7 +33,7 @@ func passLayoutBlocks(b *builder) { } nonSplitBlocks = append(nonSplitBlocks, blk) if i != len(b.reversePostOrderedBasicBlocks)-1 { - _ = maybeInvertBranches(blk, b.reversePostOrderedBasicBlocks[i+1]) + _ = maybeInvertBranches(b, blk, b.reversePostOrderedBasicBlocks[i+1]) } } @@ -111,7 +111,7 @@ func passLayoutBlocks(b *builder) { } fallthroughBranch := blk.currentInstr - if fallthroughBranch.opcode == OpcodeJump && fallthroughBranch.blk == trampoline { + if fallthroughBranch.opcode == OpcodeJump && BasicBlockID(fallthroughBranch.rValue) == trampoline.id { // This can be lowered as fallthrough at the end of the block. b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline) trampoline.visited = 1 // mark as inserted. @@ -157,7 +157,7 @@ func (b *builder) markFallthroughJumps() { for i, blk := range b.reversePostOrderedBasicBlocks { if i < l { cur := blk.currentInstr - if cur.opcode == OpcodeJump && cur.blk == b.reversePostOrderedBasicBlocks[i+1] { + if cur.opcode == OpcodeJump && BasicBlockID(cur.rValue) == b.reversePostOrderedBasicBlocks[i+1].id { cur.AsFallthroughJump() } } @@ -168,7 +168,7 @@ func (b *builder) markFallthroughJumps() { // nextInRPO is the next block in the reverse post-order. // // Returns true if the branch is inverted for testing purpose. -func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool { +func maybeInvertBranches(b *builder, now *basicBlock, nextInRPO *basicBlock) bool { fallthroughBranch := now.currentInstr if fallthroughBranch.opcode == OpcodeBrTable { return false @@ -187,7 +187,8 @@ func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool { // So this block has two branches (a conditional branch followed by an unconditional branch) at the end. // We can invert the condition of the branch if it makes the fallthrough more likely. - fallthroughTarget, condTarget := fallthroughBranch.blk.(*basicBlock), condBranch.blk.(*basicBlock) + fallthroughTarget := b.basicBlock(BasicBlockID(fallthroughBranch.rValue)) + condTarget := b.basicBlock(BasicBlockID(condBranch.rValue)) if fallthroughTarget.loopHeader { // First, if the tail's target is loopHeader, we don't need to do anything here, @@ -231,8 +232,8 @@ func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool { } condBranch.InvertBrx() - condBranch.blk = fallthroughTarget - fallthroughBranch.blk = condTarget + condBranch.rValue = Value(fallthroughTarget.ID()) + fallthroughBranch.rValue = Value(condTarget.ID()) if wazevoapi.SSALoggingEnabled { fmt.Printf("inverting branches at %d->%d and %d->%d\n", now.ID(), fallthroughTarget.ID(), now.ID(), condTarget.ID()) @@ -275,7 +276,7 @@ func (b *builder) splitCriticalEdge(pred, succ *basicBlock, predInfo *basicBlock // Replace originalBranch with the newBranch. newBranch := b.AllocateInstruction() newBranch.opcode = originalBranch.opcode - newBranch.blk = trampoline + newBranch.rValue = Value(trampoline.ID()) switch originalBranch.opcode { case OpcodeJump: case OpcodeBrz, OpcodeBrnz: @@ -303,7 +304,7 @@ func (b *builder) splitCriticalEdge(pred, succ *basicBlock, predInfo *basicBlock trampoline.validate(b) } - if len(trampoline.params) > 0 { + if len(trampoline.params.View()) > 0 { panic("trampoline should not have params") } diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go index bcf83cbf8..d906e7e35 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go @@ -15,17 +15,31 @@ // // Variable is useful to track the SSA Values of a variable in the source program, and // can be used to find the corresponding latest SSA Value via Builder.FindValue. +// +// Higher 4-bit is used to store Type for this variable. type Variable uint32 // String implements fmt.Stringer. func (v Variable) String() string { - return fmt.Sprintf("var%d", v) + return fmt.Sprintf("var%d", v&0x0fffffff) +} + +func (v Variable) setType(typ Type) Variable { + if v >= 1<<28 { + panic(fmt.Sprintf("Too large variable: %d", v)) + } + return Variable(typ)<<28 | v +} + +func (v Variable) getType() Type { + return Type(v >> 28) } // Value represents an SSA value with a type information. The relationship with Variable is 1: N (including 0), // that means there might be multiple Variable(s) for a Value. // -// Higher 32-bit is used to store Type for this value. +// 32 to 59-bit is used to store the unique identifier of the Instruction that generates this value if any. +// 60 to 63-bit is used to store Type for this value. type Value uint64 // ValueID is the lower 32bit of Value, which is the pure identifier of Value without type info. @@ -33,7 +47,7 @@ func (v Variable) String() string { const ( valueIDInvalid ValueID = math.MaxUint32 - ValueInvalid Value = Value(valueIDInvalid) + ValueInvalid = Value(valueIDInvalid) ) // Format creates a debug string for this Value using the data stored in Builder. @@ -54,7 +68,7 @@ func (v Value) formatWithType(b Builder) (ret string) { if wazevoapi.SSALoggingEnabled { // This is useful to check live value analysis bugs. if bd := b.(*builder); bd.donePostBlockLayoutPasses { id := v.ID() - ret += fmt.Sprintf("(ref=%d)", bd.valueRefCounts[id]) + ret += fmt.Sprintf("(ref=%d)", bd.valuesInfo[id].RefCount) } } return ret @@ -67,7 +81,7 @@ func (v Value) Valid() bool { // Type returns the Type of this value. func (v Value) Type() Type { - return Type(v >> 32) + return Type(v >> 60) } // ID returns the valueID of this value. @@ -77,7 +91,20 @@ func (v Value) ID() ValueID { // setType sets a type to this Value and returns the updated Value. func (v Value) setType(typ Type) Value { - return v | Value(typ)<<32 + return v | Value(typ)<<60 +} + +// setInstructionID sets an Instruction.id to this Value and returns the updated Value. +func (v Value) setInstructionID(id int) Value { + if id < 0 || uint(id) >= 1<<28 { + panic(fmt.Sprintf("Too large instruction ID: %d", id)) + } + return v | Value(id)<<32 +} + +// instructionID() returns the Instruction.id of this Value. +func (v Value) instructionID() int { + return int(v>>32) & 0x0fffffff } // Values is a slice of Value. Use this instead of []Value to reuse the underlying memory. diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go index 7177fbb4b..3fc7aa143 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go @@ -5,9 +5,7 @@ func ResetMap[K comparable, V any](m map[K]V) map[K]V { if m == nil { m = make(map[K]V) } else { - for v := range m { - delete(m, v) - } + clear(m) } return m } diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/importresolver.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/importresolver.go new file mode 100644 index 000000000..af52cc80e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/importresolver.go @@ -0,0 +1,6 @@ +package expctxkeys + +// ImportResolverKey is a context.Context Value key. +// Its associated value should be an ImportResolver. +// See issue 2294. +type ImportResolverKey struct{} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go deleted file mode 100644 index 5cba99fb2..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build !(darwin || linux || freebsd) || tinygo - -package platform - -func remapCodeSegmentAMD64(code []byte, size int) ([]byte, error) { - b, err := mmapCodeSegmentAMD64(size) - if err != nil { - return nil, err - } - copy(b, code) - mustMunmapCodeSegment(code) - return b, nil -} - -func remapCodeSegmentARM64(code []byte, size int) ([]byte, error) { - b, err := mmapCodeSegmentARM64(size) - if err != nil { - return nil, err - } - copy(b, code) - mustMunmapCodeSegment(code) - return b, nil -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go deleted file mode 100644 index 8f42d44fd..000000000 --- a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build (darwin || linux || freebsd) && !tinygo - -package platform - -func remapCodeSegmentAMD64(code []byte, size int) ([]byte, error) { - return remapCodeSegment(code, size, mmapProtAMD64) -} - -func remapCodeSegmentARM64(code []byte, size int) ([]byte, error) { - return remapCodeSegment(code, size, mmapProtARM64) -} - -func remapCodeSegment(code []byte, size, prot int) ([]byte, error) { - b, err := mmapCodeSegment(size, prot) - if err != nil { - return nil, err - } - copy(b, code) - mustMunmapCodeSegment(code) - return b, nil -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go b/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go index c6dc0f857..a27556240 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go @@ -36,28 +36,6 @@ func MmapCodeSegment(size int) ([]byte, error) { } } -// RemapCodeSegment reallocates the memory mapping of an existing code segment -// to increase its size. The previous code mapping is unmapped and must not be -// reused after the function returns. -// -// This is similar to mremap(2) on linux, and emulated on platforms which do not -// have this syscall. -// -// See https://man7.org/linux/man-pages/man2/mremap.2.html -func RemapCodeSegment(code []byte, size int) ([]byte, error) { - if size < len(code) { - panic("BUG: RemapCodeSegment with size less than code") - } - if code == nil { - return MmapCodeSegment(size) - } - if runtime.GOARCH == "amd64" { - return remapCodeSegmentAMD64(code, size) - } else { - return remapCodeSegmentARM64(code, size) - } -} - // MunmapCodeSegment unmaps the given memory region. func MunmapCodeSegment(code []byte) error { if len(code) == 0 { @@ -65,17 +43,3 @@ func MunmapCodeSegment(code []byte) error { } return munmapCodeSegment(code) } - -// mustMunmapCodeSegment panics instead of returning an error to the -// application. -// -// # Why panic? -// -// It is less disruptive to the application to leak the previous block if it -// could be unmapped than to leak the new block and return an error. -// Realistically, either scenarios are pretty hard to debug, so we panic. -func mustMunmapCodeSegment(code []byte) { - if err := munmapCodeSegment(code); err != nil { - panic(err) - } -} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go index 755ee5ea3..dfc4417ed 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go @@ -54,7 +54,6 @@ func decodeUTF8(r *bytes.Reader, contextFormat string, contextArgs ...interface{ return "", 0, fmt.Errorf("%s is not valid UTF-8", fmt.Sprintf(contextFormat, contextArgs...)) } - // TODO: use unsafe.String after flooring Go 1.20. - ret := *(*string)(unsafe.Pointer(&buf)) + ret := unsafe.String(&buf[0], int(size)) return ret, size + uint32(sizeOfSize), nil } diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go index ce2c7254d..604489228 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go @@ -451,14 +451,14 @@ functionType := &m.TypeSection[m.FunctionSection[idx]] return fmt.Errorf("read immediate: %w", err) } - list := make([]uint32, nl) + sts.ls = sts.ls[:0] for i := uint32(0); i < nl; i++ { l, n, err := leb128.DecodeUint32(br) if err != nil { return fmt.Errorf("read immediate: %w", err) } num += n - list[i] = l + sts.ls = append(sts.ls, l) } ln, n, err := leb128.DecodeUint32(br) if err != nil { @@ -511,7 +511,7 @@ functionType := &m.TypeSection[m.FunctionSection[idx]] } } - for _, l := range list { + for _, l := range sts.ls { if int(l) >= len(controlBlockStack.stack) { return fmt.Errorf("invalid l param given for %s", OpcodeBrTableName) } @@ -2003,6 +2003,8 @@ funcType := &m.TypeSection[typeIndex] type stacks struct { vs valueTypeStack cs controlBlockStack + // ls is the label slice that is reused for each br_table instruction. + ls []uint32 } func (sts *stacks) reset(functionType *FunctionType) { @@ -2012,6 +2014,7 @@ func (sts *stacks) reset(functionType *FunctionType) { sts.vs.maximumStackPointer = 0 sts.cs.stack = sts.cs.stack[:0] sts.cs.stack = append(sts.cs.stack, controlBlock{blockType: functionType}) + sts.ls = sts.ls[:0] } type controlBlockStack struct { diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go index 947b16112..8e072fd12 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go @@ -52,7 +52,8 @@ type MemoryInstance struct { definition api.MemoryDefinition // Mux is used in interpreter mode to prevent overlapping calls to atomic instructions, - // introduced with WebAssembly threads proposal. + // introduced with WebAssembly threads proposal, and in compiler mode to make memory modifications + // within Grow non-racy for the Go race detector. Mux sync.Mutex // waiters implements atomic wait and notify. It is implemented similarly to golang.org/x/sync/semaphore, @@ -227,6 +228,11 @@ func MemoryPagesToBytesNum(pages uint32) (bytesNum uint64) { // Grow implements the same method as documented on api.Memory. func (m *MemoryInstance) Grow(delta uint32) (result uint32, ok bool) { + if m.Shared { + m.Mux.Lock() + defer m.Mux.Unlock() + } + currentPages := m.Pages() if delta == 0 { return currentPages, true @@ -299,6 +305,7 @@ func PagesToUnitOfBytes(pages uint32) string { // Uses atomic write to update the length of a slice. func atomicStoreLengthAndCap(slice *[]byte, length uintptr, cap uintptr) { + //nolint:staticcheck slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice)) capPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Cap)) atomic.StoreUintptr(capPtr, cap) @@ -308,6 +315,7 @@ func atomicStoreLengthAndCap(slice *[]byte, length uintptr, cap uintptr) { // Uses atomic write to update the length of a slice. func atomicStoreLength(slice *[]byte, length uintptr) { + //nolint:staticcheck slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice)) lenPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Len)) atomic.StoreUintptr(lenPtr, length) diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go index 1db661e85..dda6e5b63 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go @@ -3,6 +3,7 @@ import ( "context" "encoding/binary" + "errors" "fmt" "sync" "sync/atomic" @@ -352,7 +353,7 @@ func (s *Store) instantiate( return nil, err } - if err = m.resolveImports(module); err != nil { + if err = m.resolveImports(ctx, module); err != nil { return nil, err } @@ -410,12 +411,22 @@ funcIdx := *module.StartSection return } -func (m *ModuleInstance) resolveImports(module *Module) (err error) { +func (m *ModuleInstance) resolveImports(ctx context.Context, module *Module) (err error) { + // Check if ctx contains an ImportResolver. + resolveImport, _ := ctx.Value(expctxkeys.ImportResolverKey{}).(experimental.ImportResolver) + for moduleName, imports := range module.ImportPerModule { var importedModule *ModuleInstance - importedModule, err = m.s.module(moduleName) - if err != nil { - return err + if resolveImport != nil { + if v := resolveImport(moduleName); v != nil { + importedModule = v.(*ModuleInstance) + } + } + if importedModule == nil { + importedModule, err = m.s.module(moduleName) + if err != nil { + return err + } } for _, i := range imports { @@ -649,20 +660,20 @@ func (s *Store) GetFunctionTypeID(t *FunctionType) (FunctionTypeID, error) { } // CloseWithExitCode implements the same method as documented on wazero.Runtime. -func (s *Store) CloseWithExitCode(ctx context.Context, exitCode uint32) (err error) { +func (s *Store) CloseWithExitCode(ctx context.Context, exitCode uint32) error { s.mux.Lock() defer s.mux.Unlock() // Close modules in reverse initialization order. + var errs []error for m := s.moduleList; m != nil; m = m.next { // If closing this module errs, proceed anyway to close the others. - if e := m.closeWithExitCode(ctx, exitCode); e != nil && err == nil { - // TODO: use multiple errors handling in Go 1.20. - err = e // first error + if err := m.closeWithExitCode(ctx, exitCode); err != nil { + errs = append(errs, err) } } s.moduleList = nil s.nameToModule = nil s.nameToModuleCap = 0 s.typeIDs = nil - return + return errors.Join(errs...) } diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go index 17c63e38e..ede3047de 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go @@ -3,8 +3,6 @@ import ( "errors" "fmt" - - "github.com/tetratelabs/wazero/api" ) // deleteModule makes the moduleName available for instantiation again. @@ -88,7 +86,7 @@ func (s *Store) registerModule(m *ModuleInstance) error { } // Module implements wazero.Runtime Module -func (s *Store) Module(moduleName string) api.Module { +func (s *Store) Module(moduleName string) *ModuleInstance { m, err := s.module(moduleName) if err != nil { return nil diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go index 3b0d3a7a6..50ba7b2b3 100644 --- a/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go +++ b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go @@ -171,7 +171,6 @@ func (d *DWARFLines) Line(instructionOffset uint64) (ret []string) { // Advance the line reader for the found position. lineReader.Seek(ln.pos) err = lineReader.Next(&le) - if err != nil { // If we reach this block, that means there's a bug in the []line creation logic above. panic("BUG: stored dwarf.LineReaderPos is invalid") diff --git a/vendor/github.com/tetratelabs/wazero/runtime.go b/vendor/github.com/tetratelabs/wazero/runtime.go index d1f0a1a31..34742289e 100644 --- a/vendor/github.com/tetratelabs/wazero/runtime.go +++ b/vendor/github.com/tetratelabs/wazero/runtime.go @@ -197,7 +197,13 @@ func (r *runtime) Module(moduleName string) api.Module { if len(moduleName) == 0 { return nil } - return r.store.Module(moduleName) + m := r.store.Module(moduleName) + if m == nil { + return nil + } else if m.Source.IsHostModule { + return hostModuleInstance{m} + } + return m } // CompileModule implements Runtime.CompileModule diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go b/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go index 583c2adb0..cc37012cf 100644 --- a/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go +++ b/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go @@ -7,9 +7,6 @@ // sysParseable is only used here as we define "supported" as being able to // parse `info.Sys()`. The above `go:build` constraints exclude 32-bit until // that's requested. -// -// TODO: When Go 1.21 is out, use the "unix" build constraint (as 1.21 makes -// our floor Go version 1.19. const sysParseable = false func statFromFileInfo(info fs.FileInfo) Stat_t { diff --git a/vendor/modules.txt b/vendor/modules.txt index 3aa63426b..9d6bbcb24 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -30,7 +30,7 @@ codeberg.org/gruf/go-fastcopy # codeberg.org/gruf/go-fastpath/v2 v2.0.0 ## explicit; go 1.14 codeberg.org/gruf/go-fastpath/v2 -# codeberg.org/gruf/go-ffmpreg v0.2.4 +# codeberg.org/gruf/go-ffmpreg v0.2.5 ## explicit; go 1.22.0 codeberg.org/gruf/go-ffmpreg/embed/ffmpeg codeberg.org/gruf/go-ffmpreg/embed/ffprobe @@ -842,8 +842,8 @@ github.com/tdewolff/parse/v2/strconv # github.com/technologize/otel-go-contrib v1.1.1 ## explicit; go 1.17 github.com/technologize/otel-go-contrib/otelginmetrics -# github.com/tetratelabs/wazero v1.7.3 -## explicit; go 1.20 +# github.com/tetratelabs/wazero v1.8.0 +## explicit; go 1.21 github.com/tetratelabs/wazero github.com/tetratelabs/wazero/api github.com/tetratelabs/wazero/experimental