languages/go
languages/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-04 23:44:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/compile, runtime/internal/atomic: intrinsify And8, Or8 on ARM64

Browse source 
Also add assembly implementation, in case intrinsics is disabled.

Change-Id: Iff0a8a8ce326651bd29f6c403f5ec08dd3629993
Reviewed-on: https://go-review.googlesource.com/28979
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>
This commit is contained in:
Cherry Zhang 2016-09-12 15:24:11 -04:00
parent 09686a5873
commit 38d35e714a
10 changed files with 142 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/cmd/compile/internal/arm64/prog.go
View file

@ -134,10 +134,12 @@ var progtable = [arm64.ALAST & obj.AMask]obj.ProgInfo{
arm64.AFMOVD & obj.AMask: {Flags: gc.SizeD | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ALDARW & obj.AMask: {Flags: gc.SizeL | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ALDAR & obj.AMask: {Flags: gc.SizeQ | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ALDAXRB & obj.AMask: {Flags: gc.SizeB | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ALDAXRW & obj.AMask: {Flags: gc.SizeL | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ALDAXR & obj.AMask: {Flags: gc.SizeQ | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ASTLRW & obj.AMask: {Flags: gc.SizeL | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ASTLR & obj.AMask: {Flags: gc.SizeQ | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ASTLXRB & obj.AMask: {Flags: gc.SizeB | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ASTLXRW & obj.AMask: {Flags: gc.SizeL | gc.LeftRead | gc.RightWrite | gc.Move},
arm64.ASTLXR & obj.AMask: {Flags: gc.SizeQ | gc.LeftRead | gc.RightWrite | gc.Move},

34
src/cmd/compile/internal/arm64/ssa.go
View file

@ -550,6 +550,35 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
p5.To.Type = obj.TYPE_REG
p5.To.Reg = out
gc.Patch(p2, p5)
case ssa.OpARM64LoweredAtomicAnd8,
ssa.OpARM64LoweredAtomicOr8:
// LDAXRB (Rarg0), Rtmp
// AND/OR Rarg1, Rtmp
// STLXRB Rtmp, (Rarg0), Rtmp
// CBNZ Rtmp, -3(PC)
r0 := gc.SSARegNum(v.Args[0])
r1 := gc.SSARegNum(v.Args[1])
p := gc.Prog(arm64.ALDAXRB)
p.From.Type = obj.TYPE_MEM
p.From.Reg = r0
p.To.Type = obj.TYPE_REG
p.To.Reg = arm64.REGTMP
p1 := gc.Prog(v.Op.Asm())
p1.From.Type = obj.TYPE_REG
p1.From.Reg = r1
p1.To.Type = obj.TYPE_REG
p1.To.Reg = arm64.REGTMP
p2 := gc.Prog(arm64.ASTLXRB)
p2.From.Type = obj.TYPE_REG
p2.From.Reg = arm64.REGTMP
p2.To.Type = obj.TYPE_MEM
p2.To.Reg = r0
p2.RegTo2 = arm64.REGTMP
p3 := gc.Prog(arm64.ACBNZ)
p3.From.Type = obj.TYPE_REG
p3.From.Reg = arm64.REGTMP
p3.To.Type = obj.TYPE_BRANCH
gc.Patch(p3, p)
case ssa.OpARM64MOVBreg,
ssa.OpARM64MOVBUreg,
ssa.OpARM64MOVHreg,
@ -770,8 +799,9 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
ssa.OpARM64STLR, ssa.OpARM64STLRW,
ssa.OpARM64LoweredAtomicExchange64, ssa.OpARM64LoweredAtomicExchange32,
ssa.OpARM64LoweredAtomicAdd64, ssa.OpARM64LoweredAtomicAdd32,
ssa.OpARM64LoweredAtomicCas64, ssa.OpARM64LoweredAtomicCas32:
// arg0 is ptr, auxint is offset
ssa.OpARM64LoweredAtomicCas64, ssa.OpARM64LoweredAtomicCas32,
ssa.OpARM64LoweredAtomicAnd8, ssa.OpARM64LoweredAtomicOr8:
// arg0 is ptr, auxint is offset (atomic ops have auxint 0)
if w.Args[0] == v.Args[0] && w.Aux == nil && w.AuxInt >= 0 && w.AuxInt < minZeroPage {
if gc.Debug_checknil != 0 && int(v.Line) > 1 {
gc.Warnl(v.Line, "removed nil check")

4
src/cmd/compile/internal/gc/ssa.go
View file

@ -2640,11 +2640,11 @@ func intrinsicInit() {
intrinsicKey{"runtime/internal/atomic", "And8"}: enableOnArch(func(s *state, n *Node) *ssa.Value {
s.vars[&memVar] = s.newValue3(ssa.OpAtomicAnd8, ssa.TypeMem, s.intrinsicArg(n, 0), s.intrinsicArg(n, 1), s.mem())
return nil
}, sys.AMD64),
}, sys.AMD64, sys.ARM64),
intrinsicKey{"runtime/internal/atomic", "Or8"}: enableOnArch(func(s *state, n *Node) *ssa.Value {
s.vars[&memVar] = s.newValue3(ssa.OpAtomicOr8, ssa.TypeMem, s.intrinsicArg(n, 0), s.intrinsicArg(n, 1), s.mem())
return nil
}, sys.AMD64),
}, sys.AMD64, sys.ARM64),
}
// aliases internal to runtime/internal/atomic

3
src/cmd/compile/internal/ssa/gen/ARM64.rules
View file

@ -491,6 +491,9 @@
(AtomicCompareAndSwap32 ptr old new_ mem) -> (LoweredAtomicCas32 ptr old new_ mem)
(AtomicCompareAndSwap64 ptr old new_ mem) -> (LoweredAtomicCas64 ptr old new_ mem)
(AtomicAnd8 ptr val mem) -> (LoweredAtomicAnd8 ptr val mem)
(AtomicOr8 ptr val mem) -> (LoweredAtomicOr8 ptr val mem)
// Optimizations
// Absorb boolean tests into block

19
src/cmd/compile/internal/ssa/gen/ARM64Ops.go
View file

@ -426,18 +426,18 @@ func init() {
{name: "InvertFlags", argLength: 1}, // reverse direction of arg0
// atomic loads.
// load from arg0. arg1=mem.
// load from arg0. arg1=mem. auxint must be zero.
// returns <value,memory> so they can be properly ordered with other loads.
{name: "LDAR", argLength: 2, reg: gpload, asm: "LDAR"},
{name: "LDARW", argLength: 2, reg: gpload, asm: "LDARW"},
// atomic stores.
// store arg1 to arg0. arg2=mem. returns memory.
// store arg1 to arg0. arg2=mem. returns memory. auxint must be zero.
{name: "STLR", argLength: 3, reg: gpstore, asm: "STLR"},
{name: "STLRW", argLength: 3, reg: gpstore, asm: "STLRW"},
// atomic exchange.
// store arg1 to arg0. arg2=mem. returns <old content of *arg0, memory>.
// store arg1 to arg0. arg2=mem. returns <old content of *arg0, memory>. auxint must be zero.
// LDAXR (Rarg0), Rout
// STLXR Rarg1, (Rarg0), Rtmp
// CBNZ Rtmp, -2(PC)
@ -445,7 +445,7 @@ func init() {
{name: "LoweredAtomicExchange32", argLength: 3, reg: gpxchg, resultNotInArgs: true},
// atomic add.
// *arg0 += arg1. arg2=mem. returns <new content of *arg0, memory>.
// *arg0 += arg1. arg2=mem. returns <new content of *arg0, memory>. auxint must be zero.
// LDAXR (Rarg0), Rout
// ADD Rarg1, Rout
// STLXR Rout, (Rarg0), Rtmp
@ -454,7 +454,7 @@ func init() {
{name: "LoweredAtomicAdd32", argLength: 3, reg: gpxchg, resultNotInArgs: true},
// atomic compare and swap.
// arg0 = pointer, arg1 = old value, arg2 = new value, arg3 = memory.
// arg0 = pointer, arg1 = old value, arg2 = new value, arg3 = memory. auxint must be zero.
// if *arg0 == arg1 {
// *arg0 = arg2
// return (true, memory)
@ -469,6 +469,15 @@ func init() {
// CSET EQ, Rout
{name: "LoweredAtomicCas64", argLength: 4, reg: gpcas, resultNotInArgs: true, clobberFlags: true},
{name: "LoweredAtomicCas32", argLength: 4, reg: gpcas, resultNotInArgs: true, clobberFlags: true},
// atomic and/or.
// *arg0 &= (|=) arg1. arg2=mem. returns memory. auxint must be zero.
// LDAXRB (Rarg0), Rtmp
// AND/OR Rarg1, Rtmp
// STLXRB Rtmp, (Rarg0), Rtmp
// CBNZ Rtmp, -3(PC)
{name: "LoweredAtomicAnd8", argLength: 3, reg: gpstore, asm: "AND"},
{name: "LoweredAtomicOr8", argLength: 3, reg: gpstore, asm: "ORR"},
}
blocks := []blockData{

24
src/cmd/compile/internal/ssa/opGen.go
View file

@ -998,6 +998,8 @@ const (
OpARM64LoweredAtomicAdd32
OpARM64LoweredAtomicCas64
OpARM64LoweredAtomicCas32
OpARM64LoweredAtomicAnd8
OpARM64LoweredAtomicOr8
OpMIPS64ADDV
OpMIPS64ADDVconst
@ -12296,6 +12298,28 @@ var opcodeTable = [...]opInfo{
},
},
},
{
name: "LoweredAtomicAnd8",
argLen: 3,
asm: arm64.AAND,
reg: regInfo{
inputs: []inputInfo{
{1, 268173311}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g
{0, 4611686019232432127}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g SP SB
},
},
},
{
name: "LoweredAtomicOr8",
argLen: 3,
asm: arm64.AORR,
reg: regInfo{
inputs: []inputInfo{
{1, 268173311}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g
{0, 4611686019232432127}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g SP SB
},
},
},
{
name: "ADDV",

38
src/cmd/compile/internal/ssa/rewriteARM64.go
View file

@ -224,6 +224,8 @@ func rewriteValueARM64(v *Value, config *Config) bool {
return rewriteValueARM64_OpAtomicAdd32(v, config)
case OpAtomicAdd64:
return rewriteValueARM64_OpAtomicAdd64(v, config)
case OpAtomicAnd8:
return rewriteValueARM64_OpAtomicAnd8(v, config)
case OpAtomicCompareAndSwap32:
return rewriteValueARM64_OpAtomicCompareAndSwap32(v, config)
case OpAtomicCompareAndSwap64:
@ -238,6 +240,8 @@ func rewriteValueARM64(v *Value, config *Config) bool {
return rewriteValueARM64_OpAtomicLoad64(v, config)
case OpAtomicLoadPtr:
return rewriteValueARM64_OpAtomicLoadPtr(v, config)
case OpAtomicOr8:
return rewriteValueARM64_OpAtomicOr8(v, config)
case OpAtomicStore32:
return rewriteValueARM64_OpAtomicStore32(v, config)
case OpAtomicStore64:
@ -9130,6 +9134,23 @@ func rewriteValueARM64_OpAtomicAdd64(v *Value, config *Config) bool {
return true
}
}
func rewriteValueARM64_OpAtomicAnd8(v *Value, config *Config) bool {
b := v.Block
_ = b
// match: (AtomicAnd8 ptr val mem)
// cond:
// result: (LoweredAtomicAnd8 ptr val mem)
for {
ptr := v.Args[0]
val := v.Args[1]
mem := v.Args[2]
v.reset(OpARM64LoweredAtomicAnd8)
v.AddArg(ptr)
v.AddArg(val)
v.AddArg(mem)
return true
}
}
func rewriteValueARM64_OpAtomicCompareAndSwap32(v *Value, config *Config) bool {
b := v.Block
_ = b
@ -9247,6 +9268,23 @@ func rewriteValueARM64_OpAtomicLoadPtr(v *Value, config *Config) bool {
return true
}
}
func rewriteValueARM64_OpAtomicOr8(v *Value, config *Config) bool {
b := v.Block
_ = b
// match: (AtomicOr8 ptr val mem)
// cond:
// result: (LoweredAtomicOr8 ptr val mem)
for {
ptr := v.Args[0]
val := v.Args[1]
mem := v.Args[2]
v.reset(OpARM64LoweredAtomicOr8)
v.AddArg(ptr)
v.AddArg(val)
v.AddArg(mem)
return true
}
}
func rewriteValueARM64_OpAtomicStore32(v *Value, config *Config) bool {
b := v.Block
_ = b

4
src/cmd/internal/obj/arm64/asm7.go
View file

@ -1830,6 +1830,8 @@ func buildop(ctxt *obj.Link) {
oprangeset(ALDXRW, t)
case ALDAXR:
oprangeset(ALDAXRB, t)
oprangeset(ALDAXRH, t)
oprangeset(ALDAXRW, t)
case ALDXP:
@ -1844,6 +1846,8 @@ func buildop(ctxt *obj.Link) {
oprangeset(ASTXRW, t)
case ASTLXR:
oprangeset(ASTLXRB, t)
oprangeset(ASTLXRH, t)
oprangeset(ASTLXRW, t)
case ASTXP:

34
src/runtime/internal/atomic/atomic_arm64.go
View file

@ -35,37 +35,11 @@ func Load64(ptr *uint64) uint64
//go:noescape
func Loadp(ptr unsafe.Pointer) unsafe.Pointer
//go:nosplit
func Or8(addr *uint8, v uint8) {
// TODO(dfc) implement this in asm.
// Align down to 4 bytes and use 32-bit CAS.
uaddr := uintptr(unsafe.Pointer(addr))
addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3))
word := uint32(v) << ((uaddr & 3) * 8) // little endian
for {
old := *addr32
if Cas(addr32, old, old|word) {
return
}
}
}
//go:noescape
func Or8(ptr *uint8, val uint8)
//go:nosplit
func And8(addr *uint8, v uint8) {
// TODO(dfc) implement this in asm.
// Align down to 4 bytes and use 32-bit CAS.
uaddr := uintptr(unsafe.Pointer(addr))
addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3))
word := uint32(v) << ((uaddr & 3) * 8) // little endian
mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian
word |= ^mask
for {
old := *addr32
if Cas(addr32, old, old&word) {
return
}
}
}
//go:noescape
func And8(ptr *uint8, val uint8)
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

19
src/runtime/internal/atomic/atomic_arm64.s
View file

@ -111,3 +111,22 @@ again:
TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-24
B runtime∕internal∕atomic·Xchg64(SB)
TEXT ·And8(SB), NOSPLIT, $0-9
MOVD ptr+0(FP), R0
MOVB val+8(FP), R1
LDAXRB (R0), R2
AND R1, R2
STLXRB R2, (R0), R3
CBNZ R3, -3(PC)
RET
TEXT ·Or8(SB), NOSPLIT, $0-9
MOVD ptr+0(FP), R0
MOVB val+8(FP), R1
LDAXRB (R0), R2
ORR R1, R2
STLXRB R2, (R0), R3
CBNZ R3, -3(PC)
RET