runtime: use vDSO clock_gettime on linux/riscv64

Speed up nanotime1 and walltime on riscv64 with vDSO, just like the other vDSO-enabled targets. name old time/op new time/op delta Now 1.53µs ± 1% 1.05µs ± 3% -31.74% (p=0.000 n=10+10) NowUnixNano 1.54µs ± 0% 1.05µs ± 2% -31.91% (p=0.000 n=9+10) NowUnixMilli 1.56µs ± 1% 1.06µs ± 1% -31.73% (p=0.000 n=10+10) NowUnixMicro 1.56µs ± 1% 1.05µs ± 1% -32.80% (p=0.000 n=9+10) Change-Id: I69b3fe3cc57685a826c53b366c0992e8048399bb Reviewed-on: https://go-review.googlesource.com/c/go/+/328509 Trust: Meng Zhuo <mzh@golangcn.org> Run-TryBot: Meng Zhuo <mzh@golangcn.org> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Cherry Mui <cherryyz@google.com>
2024-09-15 22:20:06 +00:00 · 2021-08-26 09:32:38 +08:00 · 2021-08-26 09:32:38 +08:00 · 1bd35fab05
parent df4c625d88
commit 1bd35fab05
7 changed files with 159 additions and 15 deletions
--- a/src/runtime/os_linux_novdso.go
+++ b/src/runtime/os_linux_novdso.go
@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le
-// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le
+//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64

 package runtime

--- a/src/runtime/signal_unix.go
+++ b/src/runtime/signal_unix.go
@ -383,7 +383,7 @@ func preemptM(mp *m) {
 //go:nosplit
 func sigFetchG(c *sigctxt) *g {
 	switch GOARCH {
-	case "arm", "arm64", "ppc64", "ppc64le":
+	case "arm", "arm64", "ppc64", "ppc64le", "riscv64":
 		if !iscgo && inVDSOPage(c.sigpc()) {
 			// When using cgo, we save the g on TLS and load it from there
 			// in sigtramp. Just use that.
--- a/src/runtime/sys_linux_riscv64.s
+++ b/src/runtime/sys_linux_riscv64.s
@ -10,6 +10,8 @@
 #include "go_asm.h"

 #define AT_FDCWD -100
+#define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 1

 #define SYS_brk			214
 #define SYS_clock_gettime	113
@ -210,8 +212,68 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
 	RET

 // func walltime() (sec int64, nsec int32)
-TEXT runtime·walltime(SB),NOSPLIT,$24-12
-	MOV	$0, A0 // CLOCK_REALTIME
+TEXT runtime·walltime(SB),NOSPLIT,$40-12
+	MOV	$CLOCK_REALTIME, A0
+
+	MOV	runtime·vdsoClockgettimeSym(SB), A7
+	BEQZ	A7, fallback
+	MOV	X2, S2 // S2,S3,S4 is unchanged by C code
+	MOV	g_m(g), S3 // S3 = m
+
+	// Save the old values on stack for reentrant
+	MOV	m_vdsoPC(S3), T0
+	MOV	T0, 24(X2)
+	MOV	m_vdsoSP(S3), T0
+	MOV	T0, 32(X2)
+
+	MOV	RA, m_vdsoPC(S3)
+	MOV	$ret-8(FP), T1 // caller's SP
+	MOV	T1, m_vdsoSP(S3)
+
+	MOV	m_curg(S3), T1
+	BNE	g, T1, noswitch
+
+	MOV	m_g0(S3), T1
+	MOV	(g_sched+gobuf_sp)(T1), X2
+
+noswitch:
+	ADDI	$-24, X2 // Space for result
+	ANDI	$~7, X2 // Align for C code
+	MOV	$8(X2), A1
+
+	// Store g on gsignal's stack, see sys_linux_arm64.s for detail
+	MOVBU	runtime·iscgo(SB), S4
+	BNEZ	S4, nosaveg
+	MOV	m_gsignal(S3), S4 // g.m.gsignal
+	BEQZ	S4, nosaveg
+	BEQ	g, S4, nosaveg
+	MOV	(g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo
+	MOV	g, (S4)
+
+	JALR	RA, A7
+
+	MOV	ZERO, (S4)
+	JMP	finish
+
+nosaveg:
+	JALR	RA, A7
+
+finish:
+	MOV	8(X2), T0	// sec
+	MOV	16(X2), T1	// nsec
+
+	MOV	S2, X2	// restore stack
+	MOV	24(X2), A2
+	MOV	A2, m_vdsoPC(S3)
+
+	MOV	32(X2), A3
+	MOV	A3, m_vdsoSP(S3)
+
+	MOV	T0, sec+0(FP)
+	MOVW	T1, nsec+8(FP)
+	RET
+
+fallback:
 	MOV	$8(X2), A1
 	MOV	$SYS_clock_gettime, A7
 	ECALL
@ -222,15 +284,76 @@ TEXT runtime·walltime(SB),NOSPLIT,$24-12
 	RET

 // func nanotime1() int64
-TEXT runtime·nanotime1(SB),NOSPLIT,$24-8
-	MOV	$1, A0 // CLOCK_MONOTONIC
+TEXT runtime·nanotime1(SB),NOSPLIT,$40-8
+	MOV	$CLOCK_MONOTONIC, A0
+
+	MOV	runtime·vdsoClockgettimeSym(SB), A7
+	BEQZ	A7, fallback
+
+	MOV	X2, S2 // S2 = RSP, S2 is unchanged by C code
+	MOV	g_m(g), S3 // S3 = m
+	// Save the old values on stack for reentrant
+	MOV	m_vdsoPC(S3), T0
+	MOV	T0, 24(X2)
+	MOV	m_vdsoSP(S3), T0
+	MOV	T0, 32(X2)
+
+	MOV	RA, m_vdsoPC(S3)
+	MOV	$ret-8(FP), T0 // caller's SP
+	MOV	T0, m_vdsoSP(S3)
+
+	MOV	m_curg(S3), T1
+	BNE	g, T1, noswitch
+
+	MOV	m_g0(S3), T1
+	MOV	(g_sched+gobuf_sp)(T1), X2
+
+noswitch:
+	ADDI	$-24, X2 // Space for result
+	ANDI	$~7, X2 // Align for C code
+	MOV	$8(X2), A1
+
+	// Store g on gsignal's stack, see sys_linux_arm64.s for detail
+	MOVBU	runtime·iscgo(SB), S4
+	BNEZ	S4, nosaveg
+	MOV	m_gsignal(S3), S4 // g.m.gsignal
+	BEQZ	S4, nosaveg
+	BEQ	g, S4, nosaveg
+	MOV	(g_stack+stack_lo)(S4), S4 // g.m.gsignal.stack.lo
+	MOV	g, (S4)
+
+	JALR	RA, A7
+
+	MOV	ZERO, (S4)
+	JMP	finish
+
+nosaveg:
+	JALR	RA, A7
+
+finish:
+	MOV	8(X2), T0	// sec
+	MOV	16(X2), T1	// nsec
+	// restore stack
+	MOV	S2, X2
+	MOV	24(X2), T2
+	MOV	T2, m_vdsoPC(S3)
+
+	MOV	32(X2), T2
+	MOV	T2, m_vdsoSP(S3)
+	// sec is in T0, nsec in T1
+	// return nsec in T0
+	MOV	$1000000000, T2
+	MUL	T2, T0
+	ADD	T1, T0
+	MOV	T0, ret+0(FP)
+	RET
+
+fallback:
 	MOV	$8(X2), A1
 	MOV	$SYS_clock_gettime, A7
 	ECALL
 	MOV	8(X2), T0	// sec
 	MOV	16(X2), T1	// nsec
-	// sec is in T0, nsec in T1
-	// return nsec in T0
 	MOV	$1000000000, T2
 	MUL	T2, T0
 	ADD	T1, T0
--- a/src/runtime/vdso_elf64.go
+++ b/src/runtime/vdso_elf64.go
@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le)
+//go:build linux && (amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
 // +build linux
-// +build amd64 arm64 mips64 mips64le ppc64 ppc64le
+// +build amd64 arm64 mips64 mips64le ppc64 ppc64le riscv64

 package runtime

--- a/src/runtime/vdso_in_none.go
+++ b/src/runtime/vdso_in_none.go
@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le) || !linux
-// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le !linux
+//go:build (linux && !386 && !amd64 && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !riscv64) || !linux
+// +build linux,!386,!amd64,!arm,!arm64,!mips64,!mips64le,!ppc64,!ppc64le,!riscv64 !linux

 package runtime

--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le)
+//go:build linux && (386 || amd64 || arm || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64)
 // +build linux
-// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le
+// +build 386 amd64 arm arm64 mips64 mips64le ppc64 ppc64le riscv64

 package runtime

--- a/src/runtime/vdso_linux_riscv64.go
+++ b/src/runtime/vdso_linux_riscv64.go
@ -0,0 +1,21 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+	// vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+	// See cmd/compile/internal/riscv64/galign.go arch.MAXWIDTH initialization.
+	vdsoArrayMax = 1<<50 - 1
+)
+
+// key and version at man 7 vdso : riscv
+var vdsoLinuxVersion = vdsoVersionKey{"LINUX_4.15", 0xae77f75}
+
+var vdsoSymbolKeys = []vdsoSymbolKey{
+	{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
+}
+
+// initialize to fall back to syscall
+var vdsoClockgettimeSym uintptr = 0