From f959fb3872a32d29c93f10deae31e4e6d79f2da9 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 21 Nov 2022 22:22:36 -0800 Subject: [PATCH] cmd/compile: add anchored version of SP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SPanchored opcode is identical to SP, except that it takes a memory argument so that it (and more importantly, anything that uses it) must be scheduled at or after that memory argument. This opcode ensures that a LEAQ of a variable gets scheduled after the corresponding VARDEF for that variable. This may lead to less CSE of LEAQ operations. The effect is very small. The go binary is only 80 bytes bigger after this CL. Usually LEAQs get folded into load/store operations, so the effect is only for pointerful types, large enough to need a duffzero, and have their address passed somewhere. Even then, usually the CSEd LEAQs will be un-CSEd because the two uses are on different sides of a function call and the LEAQ ends up being rematerialized at the second use anyway. Change-Id: Ib893562cd05369b91dd563b48fb83f5250950293 Reviewed-on: https://go-review.googlesource.com/c/go/+/452916 TryBot-Result: Gopher Robot Run-TryBot: Keith Randall Reviewed-by: Martin Möhrmann Reviewed-by: Martin Möhrmann Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/_gen/386.rules | 3 +- src/cmd/compile/internal/ssa/_gen/AMD64.rules | 3 +- src/cmd/compile/internal/ssa/_gen/ARM.rules | 3 +- src/cmd/compile/internal/ssa/_gen/ARM64.rules | 3 +- .../compile/internal/ssa/_gen/LOONG64.rules | 3 +- src/cmd/compile/internal/ssa/_gen/MIPS.rules | 3 +- .../compile/internal/ssa/_gen/MIPS64.rules | 3 +- src/cmd/compile/internal/ssa/_gen/PPC64.rules | 3 +- .../compile/internal/ssa/_gen/RISCV64.rules | 3 +- src/cmd/compile/internal/ssa/_gen/S390X.rules | 3 +- src/cmd/compile/internal/ssa/_gen/Wasm.rules | 3 +- .../compile/internal/ssa/_gen/genericOps.go | 7 +++-- src/cmd/compile/internal/ssa/lower.go | 2 +- src/cmd/compile/internal/ssa/opGen.go | 7 +++++ src/cmd/compile/internal/ssa/rewrite.go | 4 +-- src/cmd/compile/internal/ssa/rewrite386.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteAMD64.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteARM.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteARM64.go | 31 +++++++++++++++++-- .../compile/internal/ssa/rewriteLOONG64.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteMIPS.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteMIPS64.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewritePPC64.go | 31 +++++++++++++++++-- .../compile/internal/ssa/rewriteRISCV64.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteS390X.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteWasm.go | 31 +++++++++++++++++-- src/cmd/compile/internal/ssa/schedule.go | 29 +++++++++++++++++ test/codegen/arithmetic.go | 18 ++++++----- test/codegen/logic.go | 6 +--- 29 files changed, 395 insertions(+), 52 deletions(-) diff --git a/src/cmd/compile/internal/ssa/_gen/386.rules b/src/cmd/compile/internal/ssa/_gen/386.rules index 5e30ca9fd7..88074e5fd0 100644 --- a/src/cmd/compile/internal/ssa/_gen/386.rules +++ b/src/cmd/compile/internal/ssa/_gen/386.rules @@ -333,7 +333,8 @@ (GetCallerPC ...) => (LoweredGetCallerPC ...) (GetCallerSP ...) => (LoweredGetCallerSP ...) (Addr {sym} base) => (LEAL {sym} base) -(LocalAddr {sym} base _) => (LEAL {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (LEAL {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (LEAL {sym} base) // block rewrites (If (SETL cmp) yes no) => (LT cmp yes no) diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules index ccb5295622..c50710ec90 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules @@ -480,7 +480,8 @@ (HasCPUFeature {s}) => (SETNE (CMPLconst [0] (LoweredHasCPUFeature {s}))) (Addr {sym} base) => (LEAQ {sym} base) -(LocalAddr {sym} base _) => (LEAQ {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (LEAQ {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (LEAQ {sym} base) (MOVBstore [off] {sym} ptr y:(SETL x) mem) && y.Uses == 1 => (SETLstore [off] {sym} ptr x mem) (MOVBstore [off] {sym} ptr y:(SETLE x) mem) && y.Uses == 1 => (SETLEstore [off] {sym} ptr x mem) diff --git a/src/cmd/compile/internal/ssa/_gen/ARM.rules b/src/cmd/compile/internal/ssa/_gen/ARM.rules index e5898b0369..d8fbf41754 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM.rules +++ b/src/cmd/compile/internal/ssa/_gen/ARM.rules @@ -248,7 +248,8 @@ (OffPtr [off] ptr) => (ADDconst [int32(off)] ptr) (Addr {sym} base) => (MOVWaddr {sym} base) -(LocalAddr {sym} base _) => (MOVWaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVWaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVWaddr {sym} base) // loads (Load ptr mem) && t.IsBoolean() => (MOVBUload ptr mem) diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64.rules b/src/cmd/compile/internal/ssa/_gen/ARM64.rules index 0c5a2e66a8..0ae02f5de3 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/_gen/ARM64.rules @@ -350,7 +350,8 @@ (OffPtr [off] ptr) => (ADDconst [off] ptr) (Addr {sym} base) => (MOVDaddr {sym} base) -(LocalAddr {sym} base _) => (MOVDaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVDaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVDaddr {sym} base) // loads (Load ptr mem) && t.IsBoolean() => (MOVBUload ptr mem) diff --git a/src/cmd/compile/internal/ssa/_gen/LOONG64.rules b/src/cmd/compile/internal/ssa/_gen/LOONG64.rules index 1caaf13600..7e445e506e 100644 --- a/src/cmd/compile/internal/ssa/_gen/LOONG64.rules +++ b/src/cmd/compile/internal/ssa/_gen/LOONG64.rules @@ -227,7 +227,8 @@ (OffPtr [off] ptr) => (ADDVconst [off] ptr) (Addr {sym} base) => (MOVVaddr {sym} base) -(LocalAddr {sym} base _) => (MOVVaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVVaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVVaddr {sym} base) // loads (Load ptr mem) && t.IsBoolean() => (MOVBUload ptr mem) diff --git a/src/cmd/compile/internal/ssa/_gen/MIPS.rules b/src/cmd/compile/internal/ssa/_gen/MIPS.rules index 6f696da3cc..9cd5a1618e 100644 --- a/src/cmd/compile/internal/ssa/_gen/MIPS.rules +++ b/src/cmd/compile/internal/ssa/_gen/MIPS.rules @@ -210,7 +210,8 @@ (OffPtr [off] ptr) => (ADDconst [int32(off)] ptr) (Addr {sym} base) => (MOVWaddr {sym} base) -(LocalAddr {sym} base _) => (MOVWaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVWaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVWaddr {sym} base) // loads (Load ptr mem) && t.IsBoolean() => (MOVBUload ptr mem) diff --git a/src/cmd/compile/internal/ssa/_gen/MIPS64.rules b/src/cmd/compile/internal/ssa/_gen/MIPS64.rules index a594df2b26..b0d0dd8e66 100644 --- a/src/cmd/compile/internal/ssa/_gen/MIPS64.rules +++ b/src/cmd/compile/internal/ssa/_gen/MIPS64.rules @@ -219,7 +219,8 @@ (OffPtr [off] ptr) => (ADDVconst [off] ptr) (Addr {sym} base) => (MOVVaddr {sym} base) -(LocalAddr {sym} base _) => (MOVVaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVVaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVVaddr {sym} base) // loads (Load ptr mem) && t.IsBoolean() => (MOVBUload ptr mem) diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64.rules b/src/cmd/compile/internal/ssa/_gen/PPC64.rules index 5a68de0ca4..2eda1af3bf 100644 --- a/src/cmd/compile/internal/ssa/_gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/_gen/PPC64.rules @@ -233,7 +233,8 @@ (S(RAW|RW|LW) x (MOVDconst [c])) => (S(RAW|RW|LW)const [c&31 | (c>>5&1*31)] x) (Addr {sym} base) => (MOVDaddr {sym} [0] base) -(LocalAddr {sym} base _) => (MOVDaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVDaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVDaddr {sym} base) (OffPtr [off] ptr) => (ADD (MOVDconst [off]) ptr) // TODO: optimize these cases? diff --git a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules index 59f71be5ba..802b1dd1fd 100644 --- a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules +++ b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules @@ -547,7 +547,8 @@ (ConstBool [val]) => (MOVDconst [int64(b2i(val))]) (Addr {sym} base) => (MOVaddr {sym} [0] base) -(LocalAddr {sym} base _) => (MOVaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVaddr {sym} base) // Calls (StaticCall ...) => (CALLstatic ...) diff --git a/src/cmd/compile/internal/ssa/_gen/S390X.rules b/src/cmd/compile/internal/ssa/_gen/S390X.rules index e9becb2e17..9495010a73 100644 --- a/src/cmd/compile/internal/ssa/_gen/S390X.rules +++ b/src/cmd/compile/internal/ssa/_gen/S390X.rules @@ -446,7 +446,8 @@ (GetCallerSP ...) => (LoweredGetCallerSP ...) (GetCallerPC ...) => (LoweredGetCallerPC ...) (Addr {sym} base) => (MOVDaddr {sym} base) -(LocalAddr {sym} base _) => (MOVDaddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (MOVDaddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (MOVDaddr {sym} base) (ITab (Load ptr mem)) => (MOVDload ptr mem) // block rewrites diff --git a/src/cmd/compile/internal/ssa/_gen/Wasm.rules b/src/cmd/compile/internal/ssa/_gen/Wasm.rules index a9ed82e456..e31808ebe1 100644 --- a/src/cmd/compile/internal/ssa/_gen/Wasm.rules +++ b/src/cmd/compile/internal/ssa/_gen/Wasm.rules @@ -304,7 +304,8 @@ (GetCallerPC ...) => (LoweredGetCallerPC ...) (GetCallerSP ...) => (LoweredGetCallerSP ...) (Addr {sym} base) => (LoweredAddr {sym} [0] base) -(LocalAddr {sym} base _) => (LoweredAddr {sym} base) +(LocalAddr {sym} base mem) && t.Elem().HasPointers() => (LoweredAddr {sym} (SPanchored base mem)) +(LocalAddr {sym} base _) && !t.Elem().HasPointers() => (LoweredAddr {sym} base) // Write barrier. (WB ...) => (LoweredWB ...) diff --git a/src/cmd/compile/internal/ssa/_gen/genericOps.go b/src/cmd/compile/internal/ssa/_gen/genericOps.go index a4c8fc9c69..40c9baf5f1 100644 --- a/src/cmd/compile/internal/ssa/_gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/genericOps.go @@ -349,9 +349,10 @@ var genericOps = []opData{ {name: "Addr", argLength: 1, aux: "Sym", symEffect: "Addr"}, // Address of a variable. Arg0=SB. Aux identifies the variable. {name: "LocalAddr", argLength: 2, aux: "Sym", symEffect: "Addr"}, // Address of a variable. Arg0=SP. Arg1=mem. Aux identifies the variable. - {name: "SP", zeroWidth: true}, // stack pointer - {name: "SB", typ: "Uintptr", zeroWidth: true}, // static base pointer (a.k.a. globals pointer) - {name: "Invalid"}, // unused value + {name: "SP", zeroWidth: true}, // stack pointer + {name: "SB", typ: "Uintptr", zeroWidth: true}, // static base pointer (a.k.a. globals pointer) + {name: "Invalid"}, // unused value + {name: "SPanchored", typ: "Uintptr", argLength: 2, zeroWidth: true}, // arg0 = SP, arg1 = mem. Result is identical to arg0, but cannot be scheduled before memory state arg1. // Memory operations {name: "Load", argLength: 2}, // Load from arg0. arg1=memory diff --git a/src/cmd/compile/internal/ssa/lower.go b/src/cmd/compile/internal/ssa/lower.go index 0b79d77c50..88eb6748e8 100644 --- a/src/cmd/compile/internal/ssa/lower.go +++ b/src/cmd/compile/internal/ssa/lower.go @@ -29,7 +29,7 @@ func checkLower(f *Func) { continue // lowered } switch v.Op { - case OpSP, OpSB, OpInitMem, OpArg, OpArgIntReg, OpArgFloatReg, OpPhi, OpVarDef, OpVarLive, OpKeepAlive, OpSelect0, OpSelect1, OpSelectN, OpConvert, OpInlMark: + case OpSP, OpSPanchored, OpSB, OpInitMem, OpArg, OpArgIntReg, OpArgFloatReg, OpPhi, OpVarDef, OpVarLive, OpKeepAlive, OpSelect0, OpSelect1, OpSelectN, OpConvert, OpInlMark: continue // ok not to lower case OpMakeResult: if b.Controls[0] == v { diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9ff0ad353e..c59cfdd9ef 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -3008,6 +3008,7 @@ const ( OpLocalAddr OpSP OpSB + OpSPanchored OpLoad OpDereference OpStore @@ -38818,6 +38819,12 @@ var opcodeTable = [...]opInfo{ zeroWidth: true, generic: true, }, + { + name: "SPanchored", + argLen: 2, + zeroWidth: true, + generic: true, + }, { name: "Load", argLen: 2, diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index f4ac97c5eb..84bf2047d4 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -839,9 +839,7 @@ func isSamePtr(p1, p2 *Value) bool { case OpOffPtr: return p1.AuxInt == p2.AuxInt && isSamePtr(p1.Args[0], p2.Args[0]) case OpAddr, OpLocalAddr: - // OpAddr's 0th arg is either OpSP or OpSB, which means that it is uniquely identified by its Op. - // Checking for value equality only works after [z]cse has run. - return p1.Aux == p2.Aux && p1.Args[0].Op == p2.Args[0].Op + return p1.Aux == p2.Aux case OpAddPtr: return p1.Args[1] == p2.Args[1] && isSamePtr(p1.Args[0], p2.Args[0]) } diff --git a/src/cmd/compile/internal/ssa/rewrite386.go b/src/cmd/compile/internal/ssa/rewrite386.go index a7671e9190..064173a946 100644 --- a/src/cmd/compile/internal/ssa/rewrite386.go +++ b/src/cmd/compile/internal/ssa/rewrite386.go @@ -9050,17 +9050,44 @@ func rewriteValue386_OpLoad(v *Value) bool { return false } func rewriteValue386_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (LEAL {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (LEAL {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(Op386LEAL) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (LEAL {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(Op386LEAL) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValue386_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 395b2b1e6e..fa00bd4f5f 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -29388,17 +29388,44 @@ func rewriteValueAMD64_OpLoad(v *Value) bool { return false } func rewriteValueAMD64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (LEAQ {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (LEAQ {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpAMD64LEAQ) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (LEAQ {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpAMD64LEAQ) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueAMD64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteARM.go b/src/cmd/compile/internal/ssa/rewriteARM.go index c31d89c1fd..6ea1a7e4b2 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM.go +++ b/src/cmd/compile/internal/ssa/rewriteARM.go @@ -14047,17 +14047,44 @@ func rewriteValueARM_OpLoad(v *Value) bool { return false } func rewriteValueARM_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVWaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVWaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpARMMOVWaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVWaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpARMMOVWaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueARM_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go index 88c690be1f..6f02b507e8 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM64.go +++ b/src/cmd/compile/internal/ssa/rewriteARM64.go @@ -24790,17 +24790,44 @@ func rewriteValueARM64_OpLoad(v *Value) bool { return false } func rewriteValueARM64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVDaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVDaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpARM64MOVDaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVDaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpARM64MOVDaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueARM64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteLOONG64.go b/src/cmd/compile/internal/ssa/rewriteLOONG64.go index 451b8313eb..3c783a3037 100644 --- a/src/cmd/compile/internal/ssa/rewriteLOONG64.go +++ b/src/cmd/compile/internal/ssa/rewriteLOONG64.go @@ -4628,17 +4628,44 @@ func rewriteValueLOONG64_OpLoad(v *Value) bool { return false } func rewriteValueLOONG64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVVaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVVaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpLOONG64MOVVaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVVaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpLOONG64MOVVaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueLOONG64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteMIPS.go b/src/cmd/compile/internal/ssa/rewriteMIPS.go index f1e4970af8..a8cda7644a 100644 --- a/src/cmd/compile/internal/ssa/rewriteMIPS.go +++ b/src/cmd/compile/internal/ssa/rewriteMIPS.go @@ -1653,17 +1653,44 @@ func rewriteValueMIPS_OpLoad(v *Value) bool { return false } func rewriteValueMIPS_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVWaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVWaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpMIPSMOVWaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVWaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpMIPSMOVWaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueMIPS_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteMIPS64.go b/src/cmd/compile/internal/ssa/rewriteMIPS64.go index 14d39babf2..82d52f0b3f 100644 --- a/src/cmd/compile/internal/ssa/rewriteMIPS64.go +++ b/src/cmd/compile/internal/ssa/rewriteMIPS64.go @@ -1848,17 +1848,44 @@ func rewriteValueMIPS64_OpLoad(v *Value) bool { return false } func rewriteValueMIPS64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVVaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVVaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpMIPS64MOVVaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVVaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpMIPS64MOVVaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueMIPS64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 192ec49d82..aee570df7a 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -2352,17 +2352,44 @@ func rewriteValuePPC64_OpLoad(v *Value) bool { return false } func rewriteValuePPC64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVDaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVDaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpPPC64MOVDaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVDaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpPPC64MOVDaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValuePPC64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteRISCV64.go b/src/cmd/compile/internal/ssa/rewriteRISCV64.go index 5107b9a96f..66a6967db4 100644 --- a/src/cmd/compile/internal/ssa/rewriteRISCV64.go +++ b/src/cmd/compile/internal/ssa/rewriteRISCV64.go @@ -1607,17 +1607,44 @@ func rewriteValueRISCV64_OpLoad(v *Value) bool { return false } func rewriteValueRISCV64_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpRISCV64MOVaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpRISCV64MOVaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueRISCV64_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteS390X.go b/src/cmd/compile/internal/ssa/rewriteS390X.go index 597941c26d..b766156b43 100644 --- a/src/cmd/compile/internal/ssa/rewriteS390X.go +++ b/src/cmd/compile/internal/ssa/rewriteS390X.go @@ -2460,17 +2460,44 @@ func rewriteValueS390X_OpLoad(v *Value) bool { return false } func rewriteValueS390X_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (MOVDaddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (MOVDaddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpS390XMOVDaddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (MOVDaddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpS390XMOVDaddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueS390X_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/rewriteWasm.go b/src/cmd/compile/internal/ssa/rewriteWasm.go index 818003c4a7..bb35d8e663 100644 --- a/src/cmd/compile/internal/ssa/rewriteWasm.go +++ b/src/cmd/compile/internal/ssa/rewriteWasm.go @@ -1487,17 +1487,44 @@ func rewriteValueWasm_OpLoad(v *Value) bool { return false } func rewriteValueWasm_OpLocalAddr(v *Value) bool { + v_1 := v.Args[1] v_0 := v.Args[0] - // match: (LocalAddr {sym} base _) - // result: (LoweredAddr {sym} base) + b := v.Block + typ := &b.Func.Config.Types + // match: (LocalAddr {sym} base mem) + // cond: t.Elem().HasPointers() + // result: (LoweredAddr {sym} (SPanchored base mem)) for { + t := v.Type sym := auxToSym(v.Aux) base := v_0 + mem := v_1 + if !(t.Elem().HasPointers()) { + break + } + v.reset(OpWasmLoweredAddr) + v.Aux = symToAux(sym) + v0 := b.NewValue0(v.Pos, OpSPanchored, typ.Uintptr) + v0.AddArg2(base, mem) + v.AddArg(v0) + return true + } + // match: (LocalAddr {sym} base _) + // cond: !t.Elem().HasPointers() + // result: (LoweredAddr {sym} base) + for { + t := v.Type + sym := auxToSym(v.Aux) + base := v_0 + if !(!t.Elem().HasPointers()) { + break + } v.reset(OpWasmLoweredAddr) v.Aux = symToAux(sym) v.AddArg(base) return true } + return false } func rewriteValueWasm_OpLsh16x16(v *Value) bool { v_1 := v.Args[1] diff --git a/src/cmd/compile/internal/ssa/schedule.go b/src/cmd/compile/internal/ssa/schedule.go index 4e762f7b3f..d88c33f304 100644 --- a/src/cmd/compile/internal/ssa/schedule.go +++ b/src/cmd/compile/internal/ssa/schedule.go @@ -5,6 +5,7 @@ package ssa import ( + "cmd/compile/internal/base" "cmd/compile/internal/types" "container/heap" "sort" @@ -365,6 +366,34 @@ func schedule(f *Func) { } } + // Remove SPanchored now that we've scheduled. + for _, b := range f.Blocks { + for _, v := range b.Values { + for i, a := range v.Args { + if a.Op == OpSPanchored { + v.SetArg(i, a.Args[0]) + } + } + } + } + for _, b := range f.Blocks { + i := 0 + for _, v := range b.Values { + if v.Op == OpSPanchored { + // Free this value + if v.Uses != 0 { + base.Fatalf("SPAnchored still has %d uses", v.Uses) + } + v.resetArgs() + f.freeValue(v) + } else { + b.Values[i] = v + i++ + } + } + b.truncateValues(i) + } + f.scheduled = true } diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index 327be24db8..5139d1340a 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -316,14 +316,14 @@ func ConstMods(n1 uint, n2 int) (uint, int) { } // Check that divisibility checks x%c==0 are converted to MULs and rotates -func Divisible(n1 uint, n2 int) (bool, bool, bool, bool) { +func DivisibleU(n uint) (bool, bool) { // amd64:"MOVQ\t[$]-6148914691236517205","IMULQ","ROLQ\t[$]63",-"DIVQ" // 386:"IMUL3L\t[$]-1431655765","ROLL\t[$]31",-"DIVQ" // arm64:"MOVD\t[$]-6148914691236517205","MOVD\t[$]3074457345618258602","MUL","ROR",-"DIV" // arm:"MUL","CMP\t[$]715827882",-".*udiv" // ppc64:"MULLD","ROTL\t[$]63" // ppc64le:"MULLD","ROTL\t[$]63" - evenU := n1%6 == 0 + even := n%6 == 0 // amd64:"MOVQ\t[$]-8737931403336103397","IMULQ",-"ROLQ",-"DIVQ" // 386:"IMUL3L\t[$]678152731",-"ROLL",-"DIVQ" @@ -331,17 +331,21 @@ func Divisible(n1 uint, n2 int) (bool, bool, bool, bool) { // arm:"MUL","CMP\t[$]226050910",-".*udiv" // ppc64:"MULLD",-"ROTL" // ppc64le:"MULLD",-"ROTL" - oddU := n1%19 == 0 + odd := n%19 == 0 + return even, odd +} + +func Divisible(n int) (bool, bool) { // amd64:"IMULQ","ADD","ROLQ\t[$]63",-"DIVQ" // 386:"IMUL3L\t[$]-1431655765","ADDL\t[$]715827882","ROLL\t[$]31",-"DIVQ" - // arm64:"MUL","ADD\tR","ROR",-"DIV" + // arm64:"MOVD\t[$]-6148914691236517205","MOVD\t[$]3074457345618258602","MUL","ADD\tR","ROR",-"DIV" // arm:"MUL","ADD\t[$]715827882",-".*udiv" // ppc64/power8:"MULLD","ADD","ROTL\t[$]63" // ppc64le/power8:"MULLD","ADD","ROTL\t[$]63" // ppc64/power9:"MADDLD","ROTL\t[$]63" // ppc64le/power9:"MADDLD","ROTL\t[$]63" - evenS := n2%6 == 0 + even := n%6 == 0 // amd64:"IMULQ","ADD",-"ROLQ",-"DIVQ" // 386:"IMUL3L\t[$]678152731","ADDL\t[$]113025455",-"ROLL",-"DIVQ" @@ -351,9 +355,9 @@ func Divisible(n1 uint, n2 int) (bool, bool, bool, bool) { // ppc64/power9:"MADDLD",-"ROTL" // ppc64le/power8:"MULLD","ADD",-"ROTL" // ppc64le/power9:"MADDLD",-"ROTL" - oddS := n2%19 == 0 + odd := n%19 == 0 - return evenU, oddU, evenS, oddS + return even, odd } // Check that fix-up code is not generated for divisions where it has been proven that diff --git a/test/codegen/logic.go b/test/codegen/logic.go index 50ce5f0cca..f761e7bb9a 100644 --- a/test/codegen/logic.go +++ b/test/codegen/logic.go @@ -6,16 +6,12 @@ package codegen -var gx, gy int - // Test to make sure that (CMPQ (ANDQ x y) [0]) does not get rewritten to // (TESTQ x y) if the ANDQ has other uses. If that rewrite happens, then one // of the args of the ANDQ needs to be saved so it can be used as the arg to TESTQ. func andWithUse(x, y int) int { - // Load x,y into registers, so those MOVQ will not appear at the z := x&y line. - gx, gy = x, y - // amd64:-"MOVQ" z := x & y + // amd64:`TESTQ\s(AX, AX|BX, BX|CX, CX|DX, DX|SI, SI|DI, DI|R8, R8|R9, R9|R10, R10|R11, R11|R12, R12|R13, R13|R15, R15)` if z == 0 { return 77 }