pass: ensure frame pointer register is saved (#174)

Currently `avo` uses `BP` as a standard general-purpose register. However, `BP` is used for the frame pointer and should be callee-save. Under some circumstances, the Go assembler will do this automatically, but not always. At the moment `avo` can produce code that clobbers the `BP` register. Since Go 1.16 this code will also fail a new `go vet` check.

This PR provides a (currently sub-optimal) fix for the issue. It introduces an `EnsureBasePointerCalleeSaved` pass which will check if the base pointer is written to by a function, and if so will artificially ensure that the function has a non-zero frame size. This will trigger the Go assembler to automatically save and restore the BP register.

In addition, we update the `asmdecl` tool to `asmvet`, which includes the `framepointer` vet check.

Updates #156

pass/pass.go

@@ -21,6 +21,7 @@ var Compile = Concat(
 	FunctionPass(AllocateRegisters),
 	FunctionPass(BindRegisters),
 	FunctionPass(VerifyAllocation),
+	FunctionPass(EnsureBasePointerCalleeSaved),
 	Func(IncludeTextFlagHeader),
 	FunctionPass(PruneSelfMoves),
 	FunctionPass(RequiredISAExtensions),

pass/reg.go

@@ -3,6 +3,7 @@ package pass
 import (
 	"errors"
 
+	"github.com/mmcloughlin/avo/gotypes"
 	"github.com/mmcloughlin/avo/ir"
 	"github.com/mmcloughlin/avo/operand"
 	"github.com/mmcloughlin/avo/reg"
@@ -120,6 +121,12 @@ func BindRegisters(fn *ir.Function) error {
 		for idx := range i.Operands {
 			i.Operands[idx] = operand.ApplyAllocation(i.Operands[idx], fn.Allocation)
 		}
+		for idx := range i.Inputs {
+			i.Inputs[idx] = operand.ApplyAllocation(i.Inputs[idx], fn.Allocation)
+		}
+		for idx := range i.Outputs {
+			i.Outputs[idx] = operand.ApplyAllocation(i.Outputs[idx], fn.Allocation)
+		}
 	}
 	return nil
 }
@@ -137,3 +144,59 @@ func VerifyAllocation(fn *ir.Function) error {
 
 	return nil
 }
+
+// EnsureBasePointerCalleeSaved ensures that the base pointer register will be
+// saved and restored if it has been clobbered by the function.
+func EnsureBasePointerCalleeSaved(fn *ir.Function) error {
+	// Check to see if the base pointer is written to.
+	clobbered := false
+	for _, i := range fn.Instructions() {
+		for _, r := range i.OutputRegisters() {
+			if p := reg.ToPhysical(r); p != nil && (p.Info()&reg.BasePointer) != 0 {
+				clobbered = true
+			}
+		}
+	}
+
+	if !clobbered {
+		return nil
+	}
+
+	// This function clobbers the base pointer register so we need to ensure it
+	// will be saved and restored. The Go assembler will do this automatically,
+	// with a few exceptions detailed below. In summary, we can usually ensure
+	// this happens by ensuring the function is not frameless (apart from
+	// NOFRAME functions).
+	//
+	// Reference: https://github.com/golang/go/blob/3f4977bd5800beca059defb5de4dc64cd758cbb9/src/cmd/internal/obj/x86/obj6.go#L591-L609
+	//
+	//		var bpsize int
+	//		if ctxt.Arch.Family == sys.AMD64 &&
+	//			!p.From.Sym.NoFrame() && // (1) below
+	//			!(autoffset == 0 && p.From.Sym.NoSplit()) && // (2) below
+	//			!(autoffset == 0 && !hasCall) { // (3) below
+	//			// Make room to save a base pointer.
+	//			// There are 2 cases we must avoid:
+	//			// 1) If noframe is set (which we do for functions which tail call).
+	//			// 2) Scary runtime internals which would be all messed up by frame pointers.
+	//			//    We detect these using a heuristic: frameless nosplit functions.
+	//			//    TODO: Maybe someday we label them all with NOFRAME and get rid of this heuristic.
+	//			// For performance, we also want to avoid:
+	//			// 3) Frameless leaf functions
+	//			bpsize = ctxt.Arch.PtrSize
+	//			autoffset += int32(bpsize)
+	//			p.To.Offset += int64(bpsize)
+	//		} else {
+	//			bpsize = 0
+	//		}
+	//
+	if fn.Attributes.NOFRAME() {
+		return errors.New("NOFRAME function clobbers base pointer register")
+	}
+
+	if fn.LocalSize == 0 {
+		fn.AllocLocal(int(gotypes.PointerSize))
+	}
+
+	return nil
+}

pass/reg_test.go

@@ -3,6 +3,7 @@ package pass_test
 import (
 	"testing"
 
+	"github.com/mmcloughlin/avo/attr"
 	"github.com/mmcloughlin/avo/build"
 	"github.com/mmcloughlin/avo/ir"
 	"github.com/mmcloughlin/avo/operand"
@@ -104,3 +105,54 @@ func AssertRegistersMatchSet(t *testing.T, rs []reg.Register, s reg.MaskSet) {
 func ConstructLiveness(t *testing.T, ctx *build.Context) *ir.Function {
 	return BuildFunction(t, ctx, pass.LabelTarget, pass.CFG, pass.Liveness)
 }
+
+func TestEnsureBasePointerCalleeSavedFrameless(t *testing.T) {
+	// Construct a function that writes to the base pointer.
+	ctx := build.NewContext()
+	ctx.Function("clobber")
+	ctx.ADDQ(reg.RAX, reg.RBP)
+
+	// Build the function with the EnsureBasePointerCalleeSaved pass.
+	fn := BuildFunction(t, ctx, pass.EnsureBasePointerCalleeSaved)
+
+	// Since the function was frameless, expect that the pass would have
+	expect := 8
+	if fn.LocalSize != expect {
+		t.Fatalf("expected frame size %d; got %d", expect, fn.LocalSize)
+	}
+}
+
+func TestEnsureBasePointerCalleeSavedWithFrame(t *testing.T) {
+	// Construct a function that writes to the base pointer, but already has a
+	// stack frame.
+	expect := 64
+
+	ctx := build.NewContext()
+	ctx.Function("clobber")
+	ctx.AllocLocal(expect)
+	ctx.ADDQ(reg.RAX, reg.RBP)
+
+	// Build the function with the EnsureBasePointerCalleeSaved pass.
+	fn := BuildFunction(t, ctx, pass.EnsureBasePointerCalleeSaved)
+
+	// Expect that since the function already has a stack frame, there's no need to increase its size.
+	if fn.LocalSize != expect {
+		t.Fatalf("expected frame size %d; got %d", expect, fn.LocalSize)
+	}
+}
+
+func TestEnsureBasePointerCalleeSavedNOFRAME(t *testing.T) {
+	// Construct a NOFRAME function that writes to base pointer.
+	ctx := build.NewContext()
+	ctx.Function("clobber")
+	ctx.Attributes(attr.NOFRAME)
+	ctx.ADDQ(reg.RAX, reg.RBP)
+
+	// Build the function.
+	fn := BuildFunction(t, ctx)
+
+	// Expect the pass to fail due to NOFRAME exception.
+	if err := pass.EnsureBasePointerCalleeSaved(fn); err == nil {
+		t.Fatal("expected error from NOFRAME function that clobbers base pointer")
+	}
+}