diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 042e3f2332..594f280f03 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -179,6 +179,8 @@ func CanInline(fn *ir.Func) { Cost: inlineMaxBudget - visitor.budget, Dcl: pruneUnusedAutos(n.Defn.(*ir.Func).Dcl, &visitor), Body: inlcopylist(fn.Body), + + CanDelayResults: canDelayResults(fn), } if base.Flag.LowerM > 1 { @@ -191,6 +193,38 @@ func CanInline(fn *ir.Func) { } } +// canDelayResults reports whether inlined calls to fn can delay +// declaring the result parameter until the "return" statement. +func canDelayResults(fn *ir.Func) bool { + // We can delay declaring+initializing result parameters if: + // (1) there's exactly one "return" statement in the inlined function; + // (2) it's not an empty return statement (#44355); and + // (3) the result parameters aren't named. + + nreturns := 0 + ir.VisitList(fn.Body, func(n ir.Node) { + if n, ok := n.(*ir.ReturnStmt); ok { + nreturns++ + if len(n.Results) == 0 { + nreturns++ // empty return statement (case 2) + } + } + }) + + if nreturns != 1 { + return false // not exactly one return statement (case 1) + } + + // temporaries for return values. + for _, param := range fn.Type().Results().FieldSlice() { + if sym := types.OrigSym(param.Sym); sym != nil && !sym.IsBlank() { + return false // found a named result parameter (case 3) + } + } + + return true +} + // Inline_Flood marks n's inline body for export and recursively ensures // all called functions are marked too. func Inline_Flood(n *ir.Name, exportsym func(*ir.Name)) { @@ -740,6 +774,11 @@ var inlgen int // when producing output for debugging the compiler itself. var SSADumpInline = func(*ir.Func) {} +// NewInline allows the inliner implementation to be overridden. +// If it returns nil, the legacy inliner will handle this call +// instead. +var NewInline = func(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr { return nil } + // If n is a call node (OCALLFUNC or OCALLMETH), and fn is an ONAME node for a // function with an inlinable body, return an OINLCALL node that can replace n. // The returned node's Ninit has the parameter assignments, the Nbody is the @@ -796,30 +835,64 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b typecheck.FixVariadicCall(n) - if base.Debug.TypecheckInl == 0 { - typecheck.ImportedBody(fn) + parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex() + + sym := fn.Linksym() + inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym) + + if base.Flag.GenDwarfInl > 0 { + if !sym.WasInlined() { + base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn) + sym.Set(obj.AttrWasInlined, true) + } } - // We have a function node, and it has an inlineable body. - if base.Flag.LowerM > 1 { - fmt.Printf("%v: inlining call to %v %v { %v }\n", ir.Line(n), fn.Sym(), fn.Type(), ir.Nodes(fn.Inl.Body)) - } else if base.Flag.LowerM != 0 { + if base.Flag.LowerM != 0 { fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn) } if base.Flag.LowerM > 2 { fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n) } + res := NewInline(n, fn, inlIndex) + if res == nil { + res = oldInline(n, fn, inlIndex) + } + + // transitive inlining + // might be nice to do this before exporting the body, + // but can't emit the body with inlining expanded. + // instead we emit the things that the body needs + // and each use must redo the inlining. + // luckily these are small. + ir.EditChildren(res, edit) + + if base.Flag.LowerM > 2 { + fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res) + } + + return res +} + +// oldInline creates an InlinedCallExpr to replace the given call +// expression. fn is the callee function to be inlined. inlIndex is +// the inlining tree position index, for use with src.NewInliningBase +// when rewriting positions. +func oldInline(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr { + if base.Debug.TypecheckInl == 0 { + typecheck.ImportedBody(fn) + } + SSADumpInline(fn) - ninit := n.Init() + ninit := call.Init() // For normal function calls, the function callee expression // may contain side effects (e.g., added by addinit during // inlconv2expr or inlconv2list). Make sure to preserve these, // if necessary (#42703). - if n.Op() == ir.OCALLFUNC { - callee := n.X + if call.Op() == ir.OCALLFUNC { + callee := call.X for callee.Op() == ir.OCONVNOP { conv := callee.(*ir.ConvExpr) ninit.Append(ir.TakeInit(conv)...) @@ -857,25 +930,6 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b } // We can delay declaring+initializing result parameters if: - // (1) there's exactly one "return" statement in the inlined function; - // (2) it's not an empty return statement (#44355); and - // (3) the result parameters aren't named. - delayretvars := true - - nreturns := 0 - ir.VisitList(ir.Nodes(fn.Inl.Body), func(n ir.Node) { - if n, ok := n.(*ir.ReturnStmt); ok { - nreturns++ - if len(n.Results) == 0 { - delayretvars = false // empty return statement (case 2) - } - } - }) - - if nreturns != 1 { - delayretvars = false // not exactly one return statement (case 1) - } - // temporaries for return values. var retvars []ir.Node for i, t := range fn.Type().Results().Fields().Slice() { @@ -885,7 +939,6 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b m = inlvar(n) m = typecheck.Expr(m).(*ir.Name) inlvars[n] = m - delayretvars = false // found a named result parameter (case 3) } else { // anonymous return values, synthesize names for use in assignment that replaces return m = retvar(t, i) @@ -908,14 +961,14 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b // Assign arguments to the parameters' temp names. as := ir.NewAssignListStmt(base.Pos, ir.OAS2, nil, nil) as.Def = true - if n.Op() == ir.OCALLMETH { - sel := n.X.(*ir.SelectorExpr) + if call.Op() == ir.OCALLMETH { + sel := call.X.(*ir.SelectorExpr) if sel.X == nil { - base.Fatalf("method call without receiver: %+v", n) + base.Fatalf("method call without receiver: %+v", call) } as.Rhs.Append(sel.X) } - as.Rhs.Append(n.Args...) + as.Rhs.Append(call.Args...) if recv := fn.Type().Recv(); recv != nil { as.Lhs.Append(inlParam(recv, as, inlvars)) @@ -928,7 +981,7 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b ninit.Append(typecheck.Stmt(as)) } - if !delayretvars { + if !fn.Inl.CanDelayResults { // Zero the return parameters. for _, n := range retvars { ninit.Append(ir.NewDecl(base.Pos, ir.ODCL, n.(*ir.Name))) @@ -941,40 +994,21 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b inlgen++ - parent := -1 - if b := base.Ctxt.PosTable.Pos(n.Pos()).Base(); b != nil { - parent = b.InliningIndex() - } - - sym := fn.Linksym() - newIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym) - // Add an inline mark just before the inlined body. // This mark is inline in the code so that it's a reasonable spot // to put a breakpoint. Not sure if that's really necessary or not // (in which case it could go at the end of the function instead). // Note issue 28603. - inlMark := ir.NewInlineMarkStmt(base.Pos, types.BADWIDTH) - inlMark.SetPos(n.Pos().WithIsStmt()) - inlMark.Index = int64(newIndex) - ninit.Append(inlMark) - - if base.Flag.GenDwarfInl > 0 { - if !sym.WasInlined() { - base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn) - sym.Set(obj.AttrWasInlined, true) - } - } + ninit.Append(ir.NewInlineMarkStmt(call.Pos().WithIsStmt(), int64(inlIndex))) subst := inlsubst{ - retlabel: retlabel, - retvars: retvars, - delayretvars: delayretvars, - inlvars: inlvars, - defnMarker: ir.NilExpr{}, - bases: make(map[*src.PosBase]*src.PosBase), - newInlIndex: newIndex, - fn: fn, + retlabel: retlabel, + retvars: retvars, + inlvars: inlvars, + defnMarker: ir.NilExpr{}, + bases: make(map[*src.PosBase]*src.PosBase), + newInlIndex: inlIndex, + fn: fn, } subst.edit = subst.node @@ -995,26 +1029,11 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlMap map[*ir.Func]b //dumplist("ninit post", ninit); - call := ir.NewInlinedCallExpr(base.Pos, nil, nil) - *call.PtrInit() = ninit - call.Body = body - call.ReturnVars = retvars - call.SetType(n.Type()) - call.SetTypecheck(1) - - // transitive inlining - // might be nice to do this before exporting the body, - // but can't emit the body with inlining expanded. - // instead we emit the things that the body needs - // and each use must redo the inlining. - // luckily these are small. - ir.EditChildren(call, edit) - - if base.Flag.LowerM > 2 { - fmt.Printf("%v: After inlining %+v\n\n", ir.Line(call), call) - } - - return call + res := ir.NewInlinedCallExpr(base.Pos, body, retvars) + res.SetInit(ninit) + res.SetType(call.Type()) + res.SetTypecheck(1) + return res } // Every time we expand a function we generate a new set of tmpnames, @@ -1057,10 +1076,6 @@ type inlsubst struct { // Temporary result variables. retvars []ir.Node - // Whether result variables should be initialized at the - // "return" statement. - delayretvars bool - inlvars map[*ir.Name]*ir.Name // defnMarker is used to mark a Node for reassignment. // inlsubst.clovar set this during creating new ONAME. @@ -1353,7 +1368,7 @@ func (subst *inlsubst) node(n ir.Node) ir.Node { } as.Rhs = subst.list(n.Results) - if subst.delayretvars { + if subst.fn.Inl.CanDelayResults { for _, n := range as.Lhs { as.PtrInit().Append(ir.NewDecl(base.Pos, ir.ODCL, n.(*ir.Name))) n.Name().Defn = as diff --git a/src/cmd/compile/internal/ir/func.go b/src/cmd/compile/internal/ir/func.go index a4231a1bcb..ca6c8eca8b 100644 --- a/src/cmd/compile/internal/ir/func.go +++ b/src/cmd/compile/internal/ir/func.go @@ -166,6 +166,11 @@ type Inline struct { // another package is imported. Dcl []*Name Body []Node + + // CanDelayResults reports whether it's safe for the inliner to delay + // initializing the result parameters until immediately before the + // "return" statement. + CanDelayResults bool } // A Mark represents a scope boundary. diff --git a/src/cmd/compile/internal/typecheck/iexport.go b/src/cmd/compile/internal/typecheck/iexport.go index e798ce5143..f49718d442 100644 --- a/src/cmd/compile/internal/typecheck/iexport.go +++ b/src/cmd/compile/internal/typecheck/iexport.go @@ -1313,6 +1313,7 @@ func (w *exportWriter) funcExt(n *ir.Name) { } if n.Func.Inl != nil { w.uint64(1 + uint64(n.Func.Inl.Cost)) + w.bool(n.Func.Inl.CanDelayResults) if n.Func.ExportInline() || n.Type().HasTParam() { w.p.doInline(n) } diff --git a/src/cmd/compile/internal/typecheck/iimport.go b/src/cmd/compile/internal/typecheck/iimport.go index 16b3e7ceba..cca14a0d91 100644 --- a/src/cmd/compile/internal/typecheck/iimport.go +++ b/src/cmd/compile/internal/typecheck/iimport.go @@ -927,7 +927,8 @@ func (r *importReader) funcExt(n *ir.Name) { // Inline body. if u := r.uint64(); u > 0 { n.Func.Inl = &ir.Inline{ - Cost: int32(u - 1), + Cost: int32(u - 1), + CanDelayResults: r.bool(), } n.Func.Endlineno = r.pos() } diff --git a/test/fixedbugs/issue24651a.go b/test/fixedbugs/issue24651a.go index 6c7bf30908..1bfe8ac1ce 100644 --- a/test/fixedbugs/issue24651a.go +++ b/test/fixedbugs/issue24651a.go @@ -21,5 +21,5 @@ var x = 5 //go:noinline Provide a clean, constant reason for not inlining main func main() { // ERROR "cannot inline main: marked go:noinline$" println("Foo(", x, ")=", Foo(x)) - println("Bar(", x, ")=", Bar(x)) // ERROR "inlining call to Bar func\(int\) int { return x \* \(x \+ 1\) \* \(x \+ 2\) }$" + println("Bar(", x, ")=", Bar(x)) // ERROR "inlining call to Bar" } diff --git a/test/fixedbugs/issue24651b.go b/test/fixedbugs/issue24651b.go index aa88a6787b..2af54fc4b5 100644 --- a/test/fixedbugs/issue24651b.go +++ b/test/fixedbugs/issue24651b.go @@ -19,6 +19,6 @@ var x = 5 //go:noinline Provide a clean, constant reason for not inlining main func main() { // ERROR "cannot inline main: marked go:noinline$" - println("Foo(", x, ")=", Foo(x)) // ERROR "inlining call to Foo func\(int\) int { return x \* \(x \+ 1\) \* \(x \+ 2\) }$" - println("Bar(", x, ")=", Bar(x)) // ERROR "inlining call to Bar func\(int\) int { return x \* \(x \+ 1\) \* \(x \+ 2\) }$" + println("Foo(", x, ")=", Foo(x)) // ERROR "inlining call to Foo" + println("Bar(", x, ")=", Bar(x)) // ERROR "inlining call to Bar" } diff --git a/test/inline_big.go b/test/inline_big.go index 68e1101d3b..83672753f7 100644 --- a/test/inline_big.go +++ b/test/inline_big.go @@ -1023,7 +1023,7 @@ func f(a []int) int { // ERROR "cannot inline f:.*" "a does not escape" a[997] = 0 a[998] = 0 a[999] = 0 - x := small(a) // ERROR "inlining call to small .*" + x := small(a) // ERROR "inlining call to small" y := medium(a) // The crux of this test: medium is not inlined. return x + y }