runtime: update debug call protocol for register ABI

The debug call tests currently assume that the target Go function is
ABI0; this is clearly no longer true when we switch to the new ABI, so
make the tests set up argument register state in the debug call handler
and copy back results returned in registers.

A small snag in calling a Go function that follows the new ABI is that
the debug call protocol depends on the AX register being set to a
specific value as it bounces in and out of the handler, but this
register is part of the new register ABI, so results end up being
clobbered. Use R12 instead.

Next, the new desugaring behavior for "go" statements means that
newosproc1 must always call a function with no frame; if it takes any
arguments, it closes over them and they're passed in the context
register. Currently when debugCallWrap creates a new goroutine, it uses
newosproc1 directly and passes a non-zero-sized frame, so that needs to
be updated. To fix this, briefly use the g's param field which is
otherwise only used for channels to pass an explicitly allocated object
containing the "closed over" variables. While we could manually do the
desugaring ourselves (we cannot do so automatically because the Go
compiler prevents heap-allocated closures in the runtime), that bakes in
more ABI details in a place that really doesn't need to care about them.

Finally, there's an old bug here where the context register was set up
in CX, so technically closure calls never worked. Oops. It was otherwise
harmless for other types of calls before, but now CX is an argument
register, so now that interferes with regular calls, too.

For #40724.

Change-Id: I652c25ed56a25741bb04c24cfb603063c099edde
Reviewed-on: https://go-review.googlesource.com/c/go/+/309169
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Austin Clements <austin@google.com>
Reviewed-by: Alessandro Arzilli <alessandro.arzilli@gmail.com>
TryBot-Result: Go Bot <gobot@golang.org>
This commit is contained in:
Michael Anthony Knyszek 2021-04-10 00:05:07 +00:00 committed by Michael Knyszek
parent de7a87ef06
commit a89ace106f
10 changed files with 203 additions and 78 deletions

View file

@ -28,7 +28,7 @@ const (
FuncID_asmcgocall
FuncID_asyncPreempt
FuncID_cgocallback
FuncID_debugCallV1
FuncID_debugCallV2
FuncID_gcBgMarkWorker
FuncID_goexit
FuncID_gogo
@ -53,7 +53,7 @@ var funcIDs = map[string]FuncID{
"asmcgocall": FuncID_asmcgocall,
"asyncPreempt": FuncID_asyncPreempt,
"cgocallback": FuncID_cgocallback,
"debugCallV1": FuncID_debugCallV1,
"debugCallV2": FuncID_debugCallV2,
"gcBgMarkWorker": FuncID_gcBgMarkWorker,
"go": FuncID_rt0_go,
"goexit": FuncID_goexit,

View file

@ -231,9 +231,9 @@ ok:
CALL runtime·abort(SB) // mstart should never return
RET
// Prevent dead-code elimination of debugCallV1, which is
// Prevent dead-code elimination of debugCallV2, which is
// intended to be called by debuggers.
MOVQ $runtime·debugCallV1<ABIInternal>(SB), AX
MOVQ $runtime·debugCallV2<ABIInternal>(SB), AX
RET
// mainPC is a function value for runtime.main, to be passed to newproc.
@ -1763,7 +1763,7 @@ TEXT runtime·gcWriteBarrierR9<ABIInternal>(SB),NOSPLIT,$0
DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
// debugCallV1 is the entry point for debugger-injected function
// debugCallV2 is the entry point for debugger-injected function
// calls on running goroutines. It informs the runtime that a
// debug call has been injected and creates a call frame for the
// debugger to fill in.
@ -1776,7 +1776,7 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
// after step 2).
// 4. Save all machine registers (including flags and XMM reigsters)
// so they can be restored later by the debugger.
// 5. Set the PC to debugCallV1 and resume execution.
// 5. Set the PC to debugCallV2 and resume execution.
//
// If the goroutine is in state _Grunnable, then it's not generally
// safe to inject a call because it may return out via other runtime
@ -1786,19 +1786,19 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
//
// If the goroutine is in any other state, it's not safe to inject a call.
//
// This function communicates back to the debugger by setting RAX and
// This function communicates back to the debugger by setting R12 and
// invoking INT3 to raise a breakpoint signal. See the comments in the
// implementation for the protocol the debugger is expected to
// follow. InjectDebugCall in the runtime tests demonstrates this protocol.
//
// The debugger must ensure that any pointers passed to the function
// obey escape analysis requirements. Specifically, it must not pass
// a stack pointer to an escaping argument. debugCallV1 cannot check
// a stack pointer to an escaping argument. debugCallV2 cannot check
// this invariant.
//
// This is ABIInternal because Go code injects its PC directly into new
// goroutine stacks.
TEXT runtime·debugCallV1<ABIInternal>(SB),NOSPLIT,$152-0
TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT,$152-0
// Save all registers that may contain pointers so they can be
// conservatively scanned.
//
@ -1838,10 +1838,10 @@ TEXT runtime·debugCallV1<ABIInternal>(SB),NOSPLIT,$152-0
MOVQ AX, 0(SP)
MOVQ 16(SP), AX
MOVQ AX, 8(SP)
// Set AX to 8 and invoke INT3. The debugger should get the
// Set R12 to 8 and invoke INT3. The debugger should get the
// reason a call can't be injected from the top of the stack
// and resume execution.
MOVQ $8, AX
MOVQ $8, R12
BYTE $0xcc
JMP restore
@ -1849,17 +1849,18 @@ good:
// Registers are saved and it's safe to make a call.
// Open up a call frame, moving the stack if necessary.
//
// Once the frame is allocated, this will set AX to 0 and
// Once the frame is allocated, this will set R12 to 0 and
// invoke INT3. The debugger should write the argument
// frame for the call at SP, push the trapping PC on the
// stack, set the PC to the function to call, set RCX to point
// to the closure (if a closure call), and resume execution.
// frame for the call at SP, set up argument registers, push
// the trapping PC on the stack, set the PC to the function to
// call, set RDX to point to the closure (if a closure call),
// and resume execution.
//
// If the function returns, this will set AX to 1 and invoke
// If the function returns, this will set R12 to 1 and invoke
// INT3. The debugger can then inspect any return value saved
// on the stack at SP and resume execution again.
// on the stack at SP and in registers and resume execution again.
//
// If the function panics, this will set AX to 2 and invoke INT3.
// If the function panics, this will set R12 to 2 and invoke INT3.
// The interface{} value of the panic will be at SP. The debugger
// can inspect the panic value and resume execution again.
#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
@ -1887,16 +1888,16 @@ good:
MOVQ $debugCallFrameTooLarge<>(SB), AX
MOVQ AX, 0(SP)
MOVQ $20, 8(SP) // length of debugCallFrameTooLarge string
MOVQ $8, AX
MOVQ $8, R12
BYTE $0xcc
JMP restore
restore:
// Calls and failures resume here.
//
// Set AX to 16 and invoke INT3. The debugger should restore
// Set R12 to 16 and invoke INT3. The debugger should restore
// all registers except RIP and RSP and resume execution.
MOVQ $16, AX
MOVQ $16, R12
BYTE $0xcc
// We must not modify flags after this point.
@ -1925,9 +1926,9 @@ restore:
#define DEBUG_CALL_FN(NAME,MAXSIZE) \
TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
NO_LOCAL_POINTERS; \
MOVQ $0, AX; \
MOVQ $0, R12; \
BYTE $0xcc; \
MOVQ $1, AX; \
MOVQ $1, R12; \
BYTE $0xcc; \
RET
DEBUG_CALL_FN(debugCall32<>, 32)
@ -1950,7 +1951,7 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
MOVQ AX, 0(SP)
MOVQ val_data+8(FP), AX
MOVQ AX, 8(SP)
MOVQ $2, AX
MOVQ $2, R12
BYTE $0xcc
RET

View file

@ -9,17 +9,16 @@
// spends all of its time in the race runtime, which isn't a safe
// point.
// TODO(register args): We skip this under GOEXPERIMENT=regabidefer
// because debugCallWrap passes a non-empty frame to newproc1,
// triggering a panic.
//go:build amd64 && linux && !race && !goexperiment.regabidefer
// +build amd64,linux,!race,!goexperiment.regabidefer
//go:build amd64 && linux && !race
// +build amd64,linux,!race
package runtime_test
import (
"fmt"
"internal/abi"
"internal/goexperiment"
"math"
"os"
"regexp"
"runtime"
@ -119,21 +118,49 @@ func TestDebugCall(t *testing.T) {
g, after := startDebugCallWorker(t)
defer after()
type stackArgs struct {
x0 int
x1 float64
y0Ret int
y1Ret float64
}
// Inject a call into the debugCallWorker goroutine and test
// basic argument and result passing.
var args struct {
x int
yRet int
fn := func(x int, y float64) (y0Ret int, y1Ret float64) {
return x + 1, y + 1.0
}
fn := func(x int) (yRet int) {
return x + 1
var args *stackArgs
var regs abi.RegArgs
intRegs := regs.Ints[:]
floatRegs := regs.Floats[:]
fval := float64(42.0)
if goexperiment.RegabiArgs {
intRegs[0] = 42
floatRegs[0] = math.Float64bits(fval)
} else {
args = &stackArgs{
x0: 42,
x1: 42.0,
}
}
args.x = 42
if _, err := runtime.InjectDebugCall(g, fn, &args, debugCallTKill, false); err != nil {
if _, err := runtime.InjectDebugCall(g, fn, &regs, args, debugCallTKill, false); err != nil {
t.Fatal(err)
}
if args.yRet != 43 {
t.Fatalf("want 43, got %d", args.yRet)
var result0 int
var result1 float64
if goexperiment.RegabiArgs {
result0 = int(intRegs[0])
result1 = math.Float64frombits(floatRegs[0])
} else {
result0 = args.y0Ret
result1 = args.y1Ret
}
if result0 != 43 {
t.Errorf("want 43, got %d", result0)
}
if result1 != fval+1 {
t.Errorf("want 43, got %f", result1)
}
}
@ -158,7 +185,7 @@ func TestDebugCallLarge(t *testing.T) {
args.in[i] = i
want[i] = i + 1
}
if _, err := runtime.InjectDebugCall(g, fn, &args, debugCallTKill, false); err != nil {
if _, err := runtime.InjectDebugCall(g, fn, nil, &args, debugCallTKill, false); err != nil {
t.Fatal(err)
}
if want != args.out {
@ -171,7 +198,7 @@ func TestDebugCallGC(t *testing.T) {
defer after()
// Inject a call that performs a GC.
if _, err := runtime.InjectDebugCall(g, runtime.GC, nil, debugCallTKill, false); err != nil {
if _, err := runtime.InjectDebugCall(g, runtime.GC, nil, nil, debugCallTKill, false); err != nil {
t.Fatal(err)
}
}
@ -182,7 +209,7 @@ func TestDebugCallGrowStack(t *testing.T) {
// Inject a call that grows the stack. debugCallWorker checks
// for stack pointer breakage.
if _, err := runtime.InjectDebugCall(g, func() { growStack(nil) }, nil, debugCallTKill, false); err != nil {
if _, err := runtime.InjectDebugCall(g, func() { growStack(nil) }, nil, nil, debugCallTKill, false); err != nil {
t.Fatal(err)
}
}
@ -218,7 +245,7 @@ func TestDebugCallUnsafePoint(t *testing.T) {
runtime.Gosched()
}
_, err := runtime.InjectDebugCall(g, func() {}, nil, debugCallTKill, true)
_, err := runtime.InjectDebugCall(g, func() {}, nil, nil, debugCallTKill, true)
if msg := "call not at safe point"; err == nil || err.Error() != msg {
t.Fatalf("want %q, got %s", msg, err)
}
@ -242,7 +269,7 @@ func TestDebugCallPanic(t *testing.T) {
}()
g := <-ready
p, err := runtime.InjectDebugCall(g, func() { panic("test") }, nil, debugCallTKill, false)
p, err := runtime.InjectDebugCall(g, func() { panic("test") }, nil, nil, debugCallTKill, false)
if err != nil {
t.Fatal(err)
}

View file

@ -16,7 +16,7 @@ const (
debugCallUnsafePoint = "call not at safe point"
)
func debugCallV1()
func debugCallV2()
func debugCallPanicked(val interface{})
// debugCallCheck checks whether it is safe to inject a debugger
@ -96,7 +96,7 @@ func debugCallCheck(pc uintptr) string {
// function at PC dispatch.
//
// This must be deeply nosplit because there are untyped values on the
// stack from debugCallV1.
// stack from debugCallV2.
//
//go:nosplit
func debugCallWrap(dispatch uintptr) {
@ -108,14 +108,16 @@ func debugCallWrap(dispatch uintptr) {
// Create a new goroutine to execute the call on. Run this on
// the system stack to avoid growing our stack.
systemstack(func() {
var args struct {
dispatch uintptr
callingG *g
}
args.dispatch = dispatch
args.callingG = gp
// TODO(mknyszek): It would be nice to wrap these arguments in an allocated
// closure and start the goroutine with that closure, but the compiler disallows
// implicit closure allocation in the runtime.
fn := debugCallWrap1
newg := newproc1(*(**funcval)(unsafe.Pointer(&fn)), unsafe.Pointer(&args), int32(unsafe.Sizeof(args)), gp, callerpc)
newg := newproc1(*(**funcval)(unsafe.Pointer(&fn)), nil, 0, gp, callerpc)
args := &debugCallWrapArgs{
dispatch: dispatch,
callingG: gp,
}
newg.param = unsafe.Pointer(args)
// If the current G is locked, then transfer that
// locked-ness to the new goroutine.
@ -185,9 +187,19 @@ func debugCallWrap(dispatch uintptr) {
gp.asyncSafePoint = false
}
type debugCallWrapArgs struct {
dispatch uintptr
callingG *g
}
// debugCallWrap1 is the continuation of debugCallWrap on the callee
// goroutine.
func debugCallWrap1(dispatch uintptr, callingG *g) {
func debugCallWrap1() {
gp := getg()
args := (*debugCallWrapArgs)(gp.param)
dispatch, callingG := args.dispatch, args.callingG
gp.param = nil
// Dispatch call and trap panics.
debugCallWrap2(dispatch)

View file

@ -0,0 +1,17 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build amd64 && linux && !goexperiment.regabiargs
// +build amd64,linux
// +build !goexperiment.regabiargs
package runtime
import "internal/abi"
func storeRegArgs(dst *sigcontext, src *abi.RegArgs) {
}
func loadRegArgs(dst *abi.RegArgs, src *sigcontext) {
}

View file

@ -0,0 +1,47 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build amd64 && linux && goexperiment.regabiargs
// +build amd64,linux
// +build goexperiment.regabiargs
package runtime
import "internal/abi"
// storeRegArgs sets up argument registers in the signal
// context state from an abi.RegArgs.
//
// Both src and dst must be non-nil.
func storeRegArgs(dst *sigcontext, src *abi.RegArgs) {
dst.rax = uint64(src.Ints[0])
dst.rbx = uint64(src.Ints[1])
dst.rcx = uint64(src.Ints[2])
dst.rdi = uint64(src.Ints[3])
dst.rsi = uint64(src.Ints[4])
dst.r8 = uint64(src.Ints[5])
dst.r9 = uint64(src.Ints[6])
dst.r10 = uint64(src.Ints[7])
dst.r11 = uint64(src.Ints[8])
for i := range src.Floats {
dst.fpstate._xmm[i].element[0] = uint32(src.Floats[i] >> 0)
dst.fpstate._xmm[i].element[1] = uint32(src.Floats[i] >> 32)
}
}
func loadRegArgs(dst *abi.RegArgs, src *sigcontext) {
dst.Ints[0] = uintptr(src.rax)
dst.Ints[1] = uintptr(src.rbx)
dst.Ints[2] = uintptr(src.rcx)
dst.Ints[3] = uintptr(src.rdi)
dst.Ints[4] = uintptr(src.rsi)
dst.Ints[5] = uintptr(src.r8)
dst.Ints[6] = uintptr(src.r9)
dst.Ints[7] = uintptr(src.r10)
dst.Ints[8] = uintptr(src.r11)
for i := range dst.Floats {
dst.Floats[i] = uint64(src.fpstate._xmm[i].element[0]) << 0
dst.Floats[i] |= uint64(src.fpstate._xmm[i].element[1]) << 32
}
}

View file

@ -8,19 +8,22 @@
package runtime
import (
"internal/abi"
"runtime/internal/sys"
"unsafe"
)
// InjectDebugCall injects a debugger call to fn into g. args must be
// a pointer to a valid call frame (including arguments and return
// space) for fn, or nil. tkill must be a function that will send
// SIGTRAP to thread ID tid. gp must be locked to its OS thread and
// InjectDebugCall injects a debugger call to fn into g. regArgs must
// contain any arguments to fn that are passed in registers, according
// to the internal Go ABI. It may be nil if no arguments are passed in
// registers to fn. args must be a pointer to a valid call frame (including
// arguments and return space) for fn, or nil. tkill must be a function that
// will send SIGTRAP to thread ID tid. gp must be locked to its OS thread and
// running.
//
// On success, InjectDebugCall returns the panic value of fn or nil.
// If fn did not panic, its results will be available in args.
func InjectDebugCall(gp *g, fn, args interface{}, tkill func(tid int) error, returnOnUnsafePoint bool) (interface{}, error) {
func InjectDebugCall(gp *g, fn interface{}, regArgs *abi.RegArgs, stackArgs interface{}, tkill func(tid int) error, returnOnUnsafePoint bool) (interface{}, error) {
if gp.lockedm == 0 {
return nil, plainError("goroutine not locked to thread")
}
@ -36,7 +39,7 @@ func InjectDebugCall(gp *g, fn, args interface{}, tkill func(tid int) error, ret
}
fv := (*funcval)(f.data)
a := efaceOf(&args)
a := efaceOf(&stackArgs)
if a._type != nil && a._type.kind&kindMask != kindPtr {
return nil, plainError("args must be a pointer or nil")
}
@ -51,7 +54,7 @@ func InjectDebugCall(gp *g, fn, args interface{}, tkill func(tid int) error, ret
// gp may not be running right now, but we can still get the M
// it will run on since it's locked.
h.mp = gp.lockedm.ptr()
h.fv, h.argp, h.argSize = fv, argp, argSize
h.fv, h.regArgs, h.argp, h.argSize = fv, regArgs, argp, argSize
h.handleF = h.handle // Avoid allocating closure during signal
defer func() { testSigtrap = nil }()
@ -91,6 +94,7 @@ type debugCallHandler struct {
gp *g
mp *m
fv *funcval
regArgs *abi.RegArgs
argp unsafe.Pointer
argSize uintptr
panic interface{}
@ -120,8 +124,8 @@ func (h *debugCallHandler) inject(info *siginfo, ctxt *sigctxt, gp2 *g) bool {
h.savedRegs = *ctxt.regs()
h.savedFP = *h.savedRegs.fpstate
h.savedRegs.fpstate = nil
// Set PC to debugCallV1.
ctxt.set_rip(uint64(funcPC(debugCallV1)))
// Set PC to debugCallV2.
ctxt.set_rip(uint64(funcPC(debugCallV2)))
// Call injected. Switch to the debugCall protocol.
testSigtrap = h.handleF
case _Grunnable:
@ -153,22 +157,28 @@ func (h *debugCallHandler) handle(info *siginfo, ctxt *sigctxt, gp2 *g) bool {
return false
}
switch status := ctxt.rax(); status {
switch status := ctxt.r12(); status {
case 0:
// Frame is ready. Copy the arguments to the frame.
// Frame is ready. Copy the arguments to the frame and to registers.
sp := ctxt.rsp()
memmove(unsafe.Pointer(uintptr(sp)), h.argp, h.argSize)
if h.regArgs != nil {
storeRegArgs(ctxt.regs(), h.regArgs)
}
// Push return PC.
sp -= sys.PtrSize
ctxt.set_rsp(sp)
*(*uint64)(unsafe.Pointer(uintptr(sp))) = ctxt.rip()
// Set PC to call and context register.
ctxt.set_rip(uint64(h.fv.fn))
ctxt.regs().rcx = uint64(uintptr(unsafe.Pointer(h.fv)))
ctxt.regs().rdx = uint64(uintptr(unsafe.Pointer(h.fv)))
case 1:
// Function returned. Copy frame back out.
// Function returned. Copy frame and result registers back out.
sp := ctxt.rsp()
memmove(h.argp, unsafe.Pointer(uintptr(sp)), h.argSize)
if h.regArgs != nil {
loadRegArgs(h.regArgs, ctxt.regs())
}
case 2:
// Function panicked. Copy panic out.
sp := ctxt.rsp()
@ -191,7 +201,7 @@ func (h *debugCallHandler) handle(info *siginfo, ctxt *sigctxt, gp2 *g) bool {
// Done
notewakeup(&h.done)
default:
h.err = plainError("unexpected debugCallV1 status")
h.err = plainError("unexpected debugCallV2 status")
notewakeup(&h.done)
}
// Resume execution.

View file

@ -872,7 +872,7 @@ func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) {
}
isAsyncPreempt := frame.fn.valid() && frame.fn.funcID == funcID_asyncPreempt
isDebugCall := frame.fn.valid() && frame.fn.funcID == funcID_debugCallV1
isDebugCall := frame.fn.valid() && frame.fn.funcID == funcID_debugCallV2
if state.conservative || isAsyncPreempt || isDebugCall {
if debugScanConservative {
println("conservatively scanning function", funcname(frame.fn), "at PC", hex(frame.continpc))

View file

@ -412,14 +412,25 @@ type g struct {
stackguard0 uintptr // offset known to liblink
stackguard1 uintptr // offset known to liblink
_panic *_panic // innermost panic - offset known to liblink
_defer *_defer // innermost defer
m *m // current m; offset known to arm liblink
sched gobuf
syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
stktopsp uintptr // expected sp at top of stack, to check in traceback
param unsafe.Pointer // passed parameter on wakeup
_panic *_panic // innermost panic - offset known to liblink
_defer *_defer // innermost defer
m *m // current m; offset known to arm liblink
sched gobuf
syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
stktopsp uintptr // expected sp at top of stack, to check in traceback
// param is a generic pointer parameter field used to pass
// values in particular contexts where other storage for the
// parameter would be difficult to find. It is currently used
// in three ways:
// 1. When a channel operation wakes up a blocked goroutine, it sets param to
// point to the sudog of the completed blocking operation.
// 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed
// the GC cycle. It is unsafe to do so in any other way, because the goroutine's
// stack may have moved in the meantime.
// 3. By debugCallWrap to pass parameters to a new goroutine because allocating a
// closure in the runtime is forbidden.
param unsafe.Pointer
atomicstatus uint32
stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
goid int64

View file

@ -316,7 +316,7 @@ const (
funcID_asmcgocall
funcID_asyncPreempt
funcID_cgocallback
funcID_debugCallV1
funcID_debugCallV2
funcID_gcBgMarkWorker
funcID_goexit
funcID_gogo