cmd/compile/internal/inl: inline based on scoring when GOEXPERIMENT=newinliner

This patch changes the inliner to use callsite scores when deciding to
inline as opposed to looking only at callee cost/hairyness.

For this to work, we have to relax the inline budget cutoff as part of
CanInline to allow for the possibility that a given function might
start off with a cost of N where N > 80, but then be called from a
callsites whose score is less than 80. Once a given function F in
package P has been approved by CanInline (based on the relaxed budget)
it will then be emitted as part of the export data, meaning that other
packages importing P will need to also need to compute callsite scores
appropriately.

For a function F that calls function G, if G is marked as potentially
inlinable then the hairyness computation for F will use G's cost for
the call to G as opposed to the default call cost; for this to work
with the new scheme (given relaxed cost change described above) we
use G's cost only if it falls below inlineExtraCallCost, otherwise
just use inlineExtraCallCost.

Included in this patch are a bunch of skips and workarounds to
selected 'errorcheck' tests in the <GOROOT>/test directory to deal
with the additional "can inline" messages emitted when the new inliner
is turned on.

Change-Id: I9be5f8cd0cd8676beb4296faf80d2f6be7246335
Reviewed-on: https://go-review.googlesource.com/c/go/+/519197
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

src/cmd/compile/internal/inline/inl.go

@@ -267,9 +267,12 @@ func garbageCollectUnreferencedHiddenClosures() {
 
 // inlineBudget determines the max budget for function 'fn' prior to
 // analyzing the hairyness of the body of 'fn'. We pass in the pgo
-// profile if available, which can change the budget. If 'verbose' is
-// set, then print a remark where we boost the budget due to PGO.
-func inlineBudget(fn *ir.Func, profile *pgo.Profile, verbose bool) int32 {
+// profile if available (which can change the budget), also a
+// 'relaxed' flag, which expands the budget slightly to allow for the
+// possibility that a call to the function might have its score
+// adjusted downwards. If 'verbose' is set, then print a remark where
+// we boost the budget due to PGO.
+func inlineBudget(fn *ir.Func, profile *pgo.Profile, relaxed bool, verbose bool) int32 {
 	// Update the budget for profile-guided inlining.
 	budget := int32(inlineMaxBudget)
 	if profile != nil {
@@ -282,6 +285,9 @@ func inlineBudget(fn *ir.Func, profile *pgo.Profile, verbose bool) int32 {
 			}
 		}
 	}
+	if relaxed {
+		budget += inlineMaxBudget
+	}
 	return budget
 }
 
@@ -332,8 +338,13 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
 		cc = 1 // this appears to yield better performance than 0.
 	}
 
-	// Compute the inline budget for this function.
-	budget := inlineBudget(fn, profile, base.Debug.PGODebug > 0)
+	// Used a "relaxed" inline budget if goexperiment.NewInliner is in
+	// effect, or if we're producing a debugging dump for unit testing.
+	relaxed := goexperiment.NewInliner ||
+		(base.Debug.DumpInlFuncProps != "")
+
+	// Compute the inline budget for this func.
+	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
 
 	// At this point in the game the function we're looking at may
 	// have "stale" autos, vars that still appear in the Dcl list, but
@@ -604,8 +615,23 @@ opSwitch:
 		}
 
 		if fn := inlCallee(v.curFunc, n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
-			v.budget -= fn.Inl.Cost
-			break
+			// In the existing inliner, it makes sense to use fn.Inl.Cost
+			// here due to the fact that an "inline F everywhere if F inlinable"
+			// strategy is used. With the new inliner, however, it is not
+			// a given that we'll inline a specific callsite -- it depends
+			// on what score we assign to the callsite. For now, use the
+			// computed cost if lower than the call cost, otherwise
+			// use call cost (we can eventually do away with this when
+			// we move to the "min-heap of callsites" scheme.
+			if !goexperiment.NewInliner {
+				v.budget -= fn.Inl.Cost
+				break
+			} else {
+				if fn.Inl.Cost < inlineExtraCallCost {
+					v.budget -= fn.Inl.Cost
+					break
+				}
+			}
 		}
 
 		// Call cost for non-leaf inlining.
@@ -977,7 +1003,16 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool
 		maxCost = inlineBigFunctionMaxCost
 	}
 
-	if callee.Inl.Cost <= maxCost {
+	metric := callee.Inl.Cost
+	if goexperiment.NewInliner {
+		ok, score := inlheur.GetCallSiteScore(n)
+		if ok {
+			metric = int32(score)
+		}
+
+	}
+
+	if metric <= maxCost {
 		// Simple case. Function is already cheap enough.
 		return true, 0
 	}
@@ -1001,7 +1036,7 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool
 		return false, maxCost
 	}
 
-	if callee.Inl.Cost > inlineHotMaxBudget {
+	if metric > inlineHotMaxBudget {
 		return false, inlineHotMaxBudget
 	}
 

src/cmd/compile/internal/inline/inlheur/callsite.go

@@ -44,6 +44,14 @@ type CallSiteTab map[*ir.CallExpr]*CallSite
 // Package-level table of callsites.
 var cstab = CallSiteTab{}
 
+func GetCallSiteScore(ce *ir.CallExpr) (bool, int) {
+	cs, ok := cstab[ce]
+	if !ok {
+		return false, 0
+	}
+	return true, cs.Score
+}
+
 type CSPropBits uint32
 
 const (

test/closure3.go

@@ -1,5 +1,8 @@
 // errorcheckandrundir -0 -m -d=inlfuncswithclosures=1
 
+//go:build !goexperiment.newinliner
+// +build !goexperiment.newinliner
+
 // Copyright 2017 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/escape4.go

@@ -1,5 +1,7 @@
 // errorcheck -0 -m
 
+//go:build !goexperiment.newinliner
+
 // Copyright 2010 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/issue19261.go

@@ -1,5 +1,8 @@
 // errorcheckdir -0 -m
 
+//go:build !goexperiment.newinliner
+// +build !goexperiment.newinliner
+
 // Copyright 2017 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/issue4099.go

@@ -1,5 +1,8 @@
 // errorcheck -0 -m
 
+//go:build !goexperiment.newinliner
+// +build !goexperiment.newinliner
+
 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -20,7 +23,7 @@ func F2([]byte)
 func G() {
 	var buf1 [10]byte
 	F1(buf1[:])
-	
+
 	var buf2 [10]byte // ERROR "moved to heap: buf2"
 	F2(buf2[:])
 }

test/fixedbugs/issue42284.go

@@ -1,5 +1,8 @@
 // errorcheckdir -0 -m
 
+//go:build !goexperiment.newinliner
+// +build !goexperiment.newinliner
+
 // Copyright 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/issue7921.go

@@ -1,6 +1,7 @@
-// +build !gcflags_noopt
 // errorcheck -0 -m
 
+//go:build !gcflags_noopt && !goexperiment.newinliner
+
 // Copyright 2018 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/inline.go

@@ -1,5 +1,8 @@
 // errorcheckwithauto -0 -m -d=inlfuncswithclosures=1
 
+//go:build !goexperiment.newinliner
+// +build !goexperiment.newinliner
+
 // Copyright 2015 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/newinline.go

@@ -0,0 +1,398 @@
+// errorcheckwithauto -0 -m -d=inlfuncswithclosures=1
+
+//go:build goexperiment.newinliner
+// +build goexperiment.newinliner
+
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Test, using compiler diagnostic flags, that inlining is working.
+// Compiles but does not run.
+
+package foo
+
+import (
+	"errors"
+	"runtime"
+	"unsafe"
+)
+
+func add2(p *byte, n uintptr) *byte { // ERROR "can inline add2" "leaking param: p to result"
+	return (*byte)(add1(unsafe.Pointer(p), n)) // ERROR "inlining call to add1"
+}
+
+func add1(p unsafe.Pointer, x uintptr) unsafe.Pointer { // ERROR "can inline add1" "leaking param: p to result"
+	return unsafe.Pointer(uintptr(p) + x)
+}
+
+func f(x *byte) *byte { // ERROR "can inline f" "leaking param: x to result"
+	return add2(x, 1) // ERROR "inlining call to add2" "inlining call to add1"
+}
+
+//go:noinline
+func g(x int) int {
+	return x + 1
+}
+
+func h(x int) int { // ERROR "can inline h"
+	return x + 2
+}
+
+func i(x int) int { // ERROR "can inline i"
+	const y = 2
+	return x + y
+}
+
+func j(x int) int { // ERROR "can inline j"
+	switch {
+	case x > 0:
+		return x + 2
+	default:
+		return x + 1
+	}
+}
+
+func f2() int { // ERROR "can inline f2"
+	tmp1 := h
+	tmp2 := tmp1
+	return tmp2(0) // ERROR "inlining call to h"
+}
+
+var abc = errors.New("abc") // ERROR "inlining call to errors.New"
+
+var somethingWrong error
+
+// local closures can be inlined
+func l(x, y int) (int, int, error) { // ERROR "can inline l"
+	e := func(err error) (int, int, error) { // ERROR "can inline l.func1" "func literal does not escape" "leaking param: err to result"
+		return 0, 0, err
+	}
+	if x == y {
+		e(somethingWrong) // ERROR "inlining call to l.func1"
+	} else {
+		f := e
+		f(nil) // ERROR "inlining call to l.func1"
+	}
+	return y, x, nil
+}
+
+// any re-assignment prevents closure inlining
+func m() int {
+	foo := func() int { return 1 } // ERROR "can inline m.func1" "func literal does not escape"
+	x := foo()
+	foo = func() int { return 2 } // ERROR "can inline m.func2" "func literal does not escape"
+	return x + foo()
+}
+
+// address taking prevents closure inlining
+func n() int { // ERROR "can inline n"
+	foo := func() int { return 1 } // ERROR "can inline n.func1" "func literal does not escape"
+	bar := &foo
+	x := (*bar)() + foo()
+	return x
+}
+
+// make sure assignment inside closure is detected
+func o() int { // ERROR "can inline o"
+	foo := func() int { return 1 } // ERROR "can inline o.func1" "func literal does not escape"
+	func(x int) {                  // ERROR "can inline o.func2"
+		if x > 10 {
+			foo = func() int { return 2 } // ERROR "can inline o.func2"
+		}
+	}(11) // ERROR "func literal does not escape" "inlining call to o.func2"
+	return foo()
+}
+
+func p() int { // ERROR "can inline p"
+	return func() int { return 42 }() // ERROR "can inline p.func1" "inlining call to p.func1"
+}
+
+func q(x int) int { // ERROR "can inline q"
+	foo := func() int { return x * 2 } // ERROR "can inline q.func1" "func literal does not escape"
+	return foo()                       // ERROR "inlining call to q.func1"
+}
+
+func r(z int) int { // ERROR "can inline r"
+	foo := func(x int) int { // ERROR "can inline r.func1" "func literal does not escape"
+		return x + z
+	}
+	bar := func(x int) int { // ERROR "func literal does not escape" "can inline r.func2"
+		return x + func(y int) int { // ERROR "can inline r.func2.1" "can inline r.r.func2.func3"
+			return 2*y + x*z
+		}(x) // ERROR "inlining call to r.func2.1"
+	}
+	return foo(42) + bar(42) // ERROR "inlining call to r.func1" "inlining call to r.func2" "inlining call to r.r.func2.func3"
+}
+
+func s0(x int) int { // ERROR "can inline s0"
+	foo := func() { // ERROR "can inline s0.func1" "func literal does not escape"
+		x = x + 1
+	}
+	foo() // ERROR "inlining call to s0.func1"
+	return x
+}
+
+func s1(x int) int { // ERROR "can inline s1"
+	foo := func() int { // ERROR "can inline s1.func1" "func literal does not escape"
+		return x
+	}
+	x = x + 1
+	return foo() // ERROR "inlining call to s1.func1"
+}
+
+func switchBreak(x, y int) int { // ERROR "can inline switchBreak"
+	var n int
+	switch x {
+	case 0:
+		n = 1
+	Done:
+		switch y {
+		case 0:
+			n += 10
+			break Done
+		}
+		n = 2
+	}
+	return n
+}
+
+func switchType(x interface{}) int { // ERROR "can inline switchType" "x does not escape"
+	switch x.(type) {
+	case int:
+		return x.(int)
+	default:
+		return 0
+	}
+}
+
+// Test that switches on constant things, with constant cases, only cost anything for
+// the case that matches. See issue 50253.
+func switchConst1(p func(string)) { // ERROR "can inline switchConst" "p does not escape"
+	const c = 1
+	switch c {
+	case 0:
+		p("zero")
+	case 1:
+		p("one")
+	case 2:
+		p("two")
+	default:
+		p("other")
+	}
+}
+
+func switchConst2() string { // ERROR "can inline switchConst2"
+	switch runtime.GOOS {
+	case "linux":
+		return "Leenooks"
+	case "windows":
+		return "Windoze"
+	case "darwin":
+		return "MackBone"
+	case "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "100":
+		return "Numbers"
+	default:
+		return "oh nose!"
+	}
+}
+func switchConst3() string { // ERROR "can inline switchConst3"
+	switch runtime.GOOS {
+	case "Linux":
+		panic("Linux")
+	case "Windows":
+		panic("Windows")
+	case "Darwin":
+		panic("Darwin")
+	case "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "100":
+		panic("Numbers")
+	default:
+		return "oh nose!"
+	}
+}
+func switchConst4() { // ERROR "can inline switchConst4"
+	const intSize = 32 << (^uint(0) >> 63)
+	want := func() string { // ERROR "can inline switchConst4.func1"
+		switch intSize {
+		case 32:
+			return "32"
+		case 64:
+			return "64"
+		default:
+			panic("unreachable")
+		}
+	}() // ERROR "inlining call to switchConst4.func1"
+	_ = want
+}
+
+func inlineRangeIntoMe(data []int) { // ERROR "can inline inlineRangeIntoMe" "data does not escape"
+	rangeFunc(data, 12) // ERROR "inlining call to rangeFunc"
+}
+
+func rangeFunc(xs []int, b int) int { // ERROR "can inline rangeFunc" "xs does not escape"
+	for i, x := range xs {
+		if x == b {
+			return i
+		}
+	}
+	return -1
+}
+
+type T struct{}
+
+func (T) meth(int, int) {} // ERROR "can inline T.meth"
+
+func k() (T, int, int) { return T{}, 0, 0 } // ERROR "can inline k"
+
+func f3() { // ERROR "can inline f3"
+	T.meth(k()) // ERROR "inlining call to k" "inlining call to T.meth"
+	// ERRORAUTO "inlining call to T.meth"
+}
+
+func small1() { // ERROR "can inline small1"
+	runtime.GC()
+}
+func small2() int { // ERROR "can inline small2"
+	return runtime.GOMAXPROCS(0)
+}
+func small3(t T) { // ERROR "can inline small3"
+	t.meth2(3, 5)
+}
+func small4(t T) { // ERROR "can inline small4"
+	t.meth2(runtime.GOMAXPROCS(0), 5)
+}
+func (T) meth2(int, int) { // ERROR "can inline T.meth2"
+	runtime.GC()
+	runtime.GC()
+}
+
+// Issue #29737 - make sure we can do inlining for a chain of recursive functions
+func ee() { // ERROR "can inline ee"
+	ff(100) // ERROR "inlining call to ff" "inlining call to gg" "inlining call to hh"
+}
+
+func ff(x int) { // ERROR "can inline ff"
+	if x < 0 {
+		return
+	}
+	gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh"
+}
+func gg(x int) { // ERROR "can inline gg"
+	hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff"
+}
+func hh(x int) { // ERROR "can inline hh"
+	ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg"
+}
+
+// Issue #14768 - make sure we can inline for loops.
+func for1(fn func() bool) { // ERROR "can inline for1" "fn does not escape"
+	for {
+		if fn() {
+			break
+		} else {
+			continue
+		}
+	}
+}
+
+func for2(fn func() bool) { // ERROR "can inline for2" "fn does not escape"
+Loop:
+	for {
+		if fn() {
+			break Loop
+		} else {
+			continue Loop
+		}
+	}
+}
+
+// Issue #18493 - make sure we can do inlining of functions with a method value
+type T1 struct{}
+
+func (a T1) meth(val int) int { // ERROR "can inline T1.meth"
+	return val + 5
+}
+
+func getMeth(t1 T1) func(int) int { // ERROR "can inline getMeth"
+	return t1.meth // ERROR "t1.meth escapes to heap"
+	// ERRORAUTO "inlining call to T1.meth"
+}
+
+func ii() { // ERROR "can inline ii"
+	var t1 T1
+	f := getMeth(t1) // ERROR "inlining call to getMeth" "t1.meth does not escape"
+	_ = f(3)
+}
+
+// Issue #42194 - make sure that functions evaluated in
+// go and defer statements can be inlined.
+func gd1(int) {
+	defer gd1(gd2()) // ERROR "inlining call to gd2"
+	defer gd3()()    // ERROR "inlining call to gd3"
+	go gd1(gd2())    // ERROR "inlining call to gd2"
+	go gd3()()       // ERROR "inlining call to gd3"
+}
+
+func gd2() int { // ERROR "can inline gd2"
+	return 1
+}
+
+func gd3() func() { // ERROR "can inline gd3"
+	return ii
+}
+
+// Issue #42788 - ensure ODEREF OCONVNOP* OADDR is low cost.
+func EncodeQuad(d []uint32, x [6]float32) { // ERROR "can inline EncodeQuad" "d does not escape"
+	_ = d[:6]
+	d[0] = float32bits(x[0]) // ERROR "inlining call to float32bits"
+	d[1] = float32bits(x[1]) // ERROR "inlining call to float32bits"
+	d[2] = float32bits(x[2]) // ERROR "inlining call to float32bits"
+	d[3] = float32bits(x[3]) // ERROR "inlining call to float32bits"
+	d[4] = float32bits(x[4]) // ERROR "inlining call to float32bits"
+	d[5] = float32bits(x[5]) // ERROR "inlining call to float32bits"
+}
+
+// float32bits is a copy of math.Float32bits to ensure that
+// these tests pass with `-gcflags=-l`.
+func float32bits(f float32) uint32 { // ERROR "can inline float32bits"
+	return *(*uint32)(unsafe.Pointer(&f))
+}
+
+// Ensure OCONVNOP is zero cost.
+func Conv(v uint64) uint64 { // ERROR "can inline Conv"
+	return conv2(conv2(conv2(v))) // ERROR "inlining call to (conv1|conv2)"
+}
+func conv2(v uint64) uint64 { // ERROR "can inline conv2"
+	return conv1(conv1(conv1(conv1(v)))) // ERROR "inlining call to conv1"
+}
+func conv1(v uint64) uint64 { // ERROR "can inline conv1"
+	return uint64(uint64(uint64(uint64(uint64(uint64(uint64(uint64(uint64(uint64(uint64(v)))))))))))
+}
+
+func select1(x, y chan bool) int { // ERROR "can inline select1" "x does not escape" "y does not escape"
+	select {
+	case <-x:
+		return 1
+	case <-y:
+		return 2
+	}
+}
+
+func select2(x, y chan bool) { // ERROR "can inline select2" "x does not escape" "y does not escape"
+loop: // test that labeled select can be inlined.
+	select {
+	case <-x:
+		break loop
+	case <-y:
+	}
+}
+
+func inlineSelect2(x, y chan bool) { // ERROR "can inline inlineSelect2" ERROR "x does not escape" "y does not escape"
+loop:
+	for i := 0; i < 5; i++ {
+		if i == 3 {
+			break loop
+		}
+		select2(x, y) // ERROR "inlining call to select2"
+	}
+}