Merge remote-tracking branch 'upstream/master' into chore/github

api/next/50489.txt

@@ -0,0 +1 @@
+pkg math/big, method (*Rat) FloatPrec() (int, bool) #50489

api/next/57178.txt

@@ -0,0 +1 @@
+pkg crypto/x509, method (*CertPool) AddCertWithConstraint(*Certificate, func([]*Certificate) error) #57178

api/next/58808.txt

@@ -0,0 +1,2 @@
+pkg net, method (*TCPConn) WriteTo(io.Writer) (int64, error) #58808
+pkg os, method (*File) WriteTo(io.Writer) (int64, error) #58808

api/next/62037.txt

@@ -0,0 +1,2 @@
+pkg go/types, method (*Info) PkgNameOf(*ast.ImportSpec) *PkgName #62037
+pkg go/types, method (Checker) PkgNameOf(*ast.ImportSpec) *PkgName #62037

api/next/62039.txt

@@ -0,0 +1,3 @@
+pkg go/version, func Compare(string, string) int #62039
+pkg go/version, func IsValid(string) bool #62039
+pkg go/version, func Lang(string) string #62039

api/next/62418.txt

@@ -0,0 +1 @@
+pkg log/slog, func SetLogLoggerLevel(Level) Level #62418

api/next/62605.txt

@@ -0,0 +1 @@
+pkg go/types, type Info struct, FileVersions map[*ast.File]string #62605

api/next/63223.txt

@@ -0,0 +1,6 @@
+pkg go/types, func NewAlias(*TypeName, Type) *Alias #63223
+pkg go/types, func Unalias(Type) Type #63223
+pkg go/types, method (*Alias) Obj() *TypeName #63223
+pkg go/types, method (*Alias) String() string #63223
+pkg go/types, method (*Alias) Underlying() Type #63223
+pkg go/types, type Alias struct #63223

doc/godebug.md

@@ -143,6 +143,27 @@ patterns and unescape both patterns and request paths by segment.
 This behavior can be controlled by the
 [`httpmuxgo121` setting](/pkg/net/http/#ServeMux).
 
+Go 1.22 added the [Alias type](/pkg/go/types#Alias) to [go/types](/pkg/go/types)
+for the explicit representation of [type aliases](/ref/spec#Type_declarations).
+Whether the type checker produces `Alias` types or not is controlled by the
+[`gotypesalias` setting](/pkg/go/types#Alias).
+For Go 1.22 it defaults to `gotypesalias=0`.
+For Go 1.23, `gotypealias=1` will become the default.
+This setting will be removed in a future release, Go 1.24 at the earliest.
+
+Go 1.22 changed the default minimum TLS version supported by both servers
+and clients to TLS 1.2. The default can be reverted to TLS 1.0 using the
+[`tls10server` setting](/pkg/crypto/tls/#Config).
+
+Go 1.22 changed the default TLS cipher suites used by clients and servers when
+not explicitly configured, removing the cipher suites which used RSA based key
+exchange. The default can be revert using the [`tlsrsakex` setting](/pkg/crypto/tls/#Config).
+
+Go 1.22 disabled
+[`ConnectionState.ExportKeyingMaterial`](/pkg/crypto/tls/#ConnectionState.ExportKeyingMaterial)
+when the connection supports neither TLS 1.3 nor Extended Master Secret
+(implemented in Go 1.21). It can be reenabled with the [`tlsunsafeekm`
+setting](/pkg/crypto/tls/#ConnectionState.ExportKeyingMaterial).
 
 ### Go 1.21
 

misc/wasm/go_wasip1_wasm_exec

@@ -14,8 +14,15 @@ case "$GOWASIRUNTIME" in
 		exec wazero run -mount /:/ -env-inherit -cachedir "${TMPDIR:-/tmp}"/wazero ${GOWASIRUNTIMEARGS:-} "$1" "${@:2}"
 		;;
 	"wasmtime" | "")
-		# TODO(go.dev/issue/63718): Switch to the new CLI offered in the major version 14 of Wasmtime.
-		exec env WASMTIME_NEW_CLI=0 wasmtime run --dir=/ --env PWD="$PWD" --env PATH="$PATH" --max-wasm-stack 1048576 ${GOWASIRUNTIMEARGS:-} "$1" -- "${@:2}"
+		# Match the major version in "wasmtime-cli 14.0.0". For versions before 14
+		# we need to use the old CLI. This requires Bash v3.0 and above.
+		# TODO(johanbrandhorst): Remove this condition once 1.22 is released.
+		# From 1.23 onwards we'll only support the new wasmtime CLI.
+		if [[ "$(wasmtime --version)" =~ wasmtime-cli[[:space:]]([0-9]+)\.[0-9]+\.[0-9]+ && "${BASH_REMATCH[1]}" -lt 14 ]]; then
+			exec wasmtime run --dir=/ --env PWD="$PWD" --env PATH="$PATH" --max-wasm-stack 1048576 ${GOWASIRUNTIMEARGS:-} "$1" -- "${@:2}"
+		else
+			exec wasmtime run --dir=/ --env PWD="$PWD" --env PATH="$PATH" -W max-wasm-stack=1048576 ${GOWASIRUNTIMEARGS:-} "$1" "${@:2}"
+		fi
 		;;
 	*)
 		echo "Unknown Go WASI runtime specified: $GOWASIRUNTIME"

src/cmd/api/main_test.go

@@ -106,7 +106,7 @@ func Check(t *testing.T) {
 	}
 
 	var nextFiles []string
-	if strings.Contains(runtime.Version(), "devel") {
+	if v := runtime.Version(); strings.Contains(v, "devel") || strings.Contains(v, "beta") {
 		next, err := filepath.Glob(filepath.Join(testenv.GOROOT(t), "api/next/*.txt"))
 		if err != nil {
 			t.Fatal(err)

src/cmd/asm/internal/asm/endtoend_test.go

@@ -372,10 +372,10 @@ func Test386EndToEnd(t *testing.T) {
 }
 
 func TestARMEndToEnd(t *testing.T) {
-	defer func(old int) { buildcfg.GOARM = old }(buildcfg.GOARM)
+	defer func(old int) { buildcfg.GOARM.Version = old }(buildcfg.GOARM.Version)
 	for _, goarm := range []int{5, 6, 7} {
 		t.Logf("GOARM=%d", goarm)
-		buildcfg.GOARM = goarm
+		buildcfg.GOARM.Version = goarm
 		testEndToEnd(t, "arm", "arm")
 		if goarm == 6 {
 			testEndToEnd(t, "arm", "armv6")

src/cmd/asm/internal/asm/testdata/ppc64.s

@@ -17,14 +17,14 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	MOVD $1, R3                     // 38600001
 	MOVD $-1, R4                    // 3880ffff
 	MOVD $65535, R5                 // 6005ffff
-	MOVD $65536, R6                 // 64060001
+	MOVD $65536, R6                 // 3cc00001
 	MOVD $-32767, R5                // 38a08001
 	MOVD $-32768, R6                // 38c08000
 	MOVD $1234567, R5               // 6405001260a5d687 or 0600001238a0d687
 	MOVW $1, R3                     // 38600001
 	MOVW $-1, R4                    // 3880ffff
 	MOVW $65535, R5                 // 6005ffff
-	MOVW $65536, R6                 // 64060001
+	MOVW $65536, R6                 // 3cc00001
 	MOVW $-32767, R5                // 38a08001
 	MOVW $-32768, R6                // 38c08000
 	MOVW $1234567, R5               // 6405001260a5d687 or 0600001238a0d687
@@ -36,6 +36,11 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	// Hex constant 0xFFFFFFFE00000002 (load of constant on < power10, pli on >= power10
 	MOVD $-8589934590, R5           // 3ca00000e8a50000 or 0602000038a00002
 
+	// For backwards compatibility, MOVW $const,Rx and MOVWZ $const,Rx assemble identically
+	// and accept the same constants.
+	MOVW $2147483648, R5            // 64058000
+	MOVWZ $-2147483648, R5          // 3ca08000
+
 	// TODO: These are preprocessed by the assembler into MOVD $const>>shift, R5; SLD $shift, R5.
 	//       This only captures the MOVD. Should the SLD be appended to the encoding by the test?
 	// Hex constant 0x20004000000
@@ -192,6 +197,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	ADDEX R3, R5, $3, R6            // 7cc32f54
 	ADDEX R3, $3, R5, R6            // 7cc32f54
 	ADDIS $8, R3                    // 3c630008
+	ADD   $524288, R3               // 3c630008
 	ADDIS $1000, R3, R4             // 3c8303e8
 
 	ANDCC $1, R3                    // 70630001
@@ -210,6 +216,7 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	ANDCC $1234567, R5, R6          // 641f001263ffd6877fe62839
 	ANDISCC $1, R3                  // 74630001
 	ANDISCC $1000, R3, R4           // 746403e8
+	ANDCC $65536000, R3, R4         // 746403e8
 
 	OR $1, R3                       // 60630001
 	OR $1, R3, R4                   // 60640001
@@ -225,9 +232,10 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	OR $-32768, R6, R7              // 3be080007fe73378
 	OR $1234567, R5                 // 641f001263ffd6877fe52b78
 	OR $1234567, R5, R3             // 641f001263ffd6877fe32b78
-	OR $2147483648, R5, R3          // 641f8000600000007fe32b78
+	OR $2147483648, R5, R3          // 64a38000
 	OR $2147483649, R5, R3          // 641f800063ff00017fe32b78
-	ORIS $255, R3, R4
+	ORIS $255, R3, R4               // 646400ff
+	OR $16711680, R3, R4            // 646400ff
 
 	XOR $1, R3                      // 68630001
 	XOR $1, R3, R4                  // 68640001
@@ -243,7 +251,8 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
 	XOR $-32768, R6, R7             // 3be080007fe73278
 	XOR $1234567, R5                // 641f001263ffd6877fe52a78
 	XOR $1234567, R5, R3            // 641f001263ffd6877fe32a78
-	XORIS $15, R3, R4
+	XORIS $15, R3, R4               // 6c64000f
+	XOR   $983040, R3, R4           // 6c64000f
 
 	// TODO: the order of CR operands don't match
 	CMP R3, R4                      // 7c232000

src/cmd/asm/internal/asm/testdata/riscv64.s

@@ -94,6 +94,10 @@ start:
 
 	SUB	X6, X5, X7				// b3836240
 	SUB	X5, X6					// 33035340
+	SUB	$-2047, X5, X6				// 1383f27f
+	SUB	$2048, X5, X6				// 13830280
+	SUB	$-2047, X5				// 9382f27f
+	SUB	$2048, X5				// 93820280
 
 	SRA	X6, X5, X7				// b3d36240
 	SRA	X5, X6					// 33535340
@@ -157,6 +161,7 @@ start:
 	ADDW	$1, X6					// 1b031300
 	SLLW	$1, X6					// 1b131300
 	SRLW	$1, X6					// 1b531300
+	SUBW	$1, X6					// 1b03f3ff
 	SRAW	$1, X6					// 1b531340
 
 	// 5.3: Load and Store Instructions (RV64I)

src/cmd/asm/internal/asm/testdata/riscv64error.s

@@ -38,5 +38,8 @@ TEXT errors(SB),$0
 	SLLIW	$-1, X5, X6			// ERROR "shift amount out of range 0 to 31"
 	SRLIW	$-1, X5, X6			// ERROR "shift amount out of range 0 to 31"
 	SRAIW	$-1, X5, X6			// ERROR "shift amount out of range 0 to 31"
-
+	SD	X5, 4294967296(X6)		// ERROR "constant 4294967296 too large"
+	SRLI	$1, X5, F1			// ERROR "expected integer register in rd position but got non-integer register F1"
+	SRLI	$1, F1, X5			// ERROR "expected integer register in rs1 position but got non-integer register F1"
+	FNES	F1, (X5)			// ERROR "needs an integer register output"
 	RET

src/cmd/asm/internal/asm/testdata/s390x.s

@@ -419,9 +419,9 @@ TEXT main·foo(SB),DUPOK|NOSPLIT,$16-0 // TEXT main.foo(SB), DUPOK|NOSPLIT, $16-
 	KMC	R2, R6                 // b92f0026
 	KLMD	R2, R8                 // b93f0028
 	KIMD	R0, R4                 // b93e0004
-	KDSA    R0, R8                 // b93a0008
-	KMA	R6, R2, R4		// b9296024
-	KMCTR	R6, R2, R4		// b92d6024
+	KDSA	R0, R8                 // b93a0008
+	KMA	R2, R6, R4              // b9296024
+	KMCTR   R2, R6, R4              // b92d6024
 
 	// vector add and sub instructions
 	VAB	V3, V4, V4              // e743400000f3

src/cmd/cgo/internal/test/callback_windows.go

@@ -0,0 +1,133 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cgotest
+
+/*
+#include <windows.h>
+USHORT backtrace(ULONG FramesToCapture, PVOID *BackTrace) {
+#ifdef _AMD64_
+	CONTEXT context;
+	RtlCaptureContext(&context);
+	ULONG64 ControlPc;
+	ControlPc = context.Rip;
+	int i;
+	for (i = 0; i < FramesToCapture; i++) {
+		PRUNTIME_FUNCTION FunctionEntry;
+		ULONG64 ImageBase;
+		VOID *HandlerData;
+		ULONG64 EstablisherFrame;
+
+		FunctionEntry = RtlLookupFunctionEntry(ControlPc, &ImageBase, NULL);
+
+		if (!FunctionEntry) {
+			// For simplicity, don't unwind leaf entries, which are not used in this test.
+			break;
+		} else {
+			RtlVirtualUnwind(0, ImageBase, ControlPc, FunctionEntry, &context, &HandlerData, &EstablisherFrame, NULL);
+		}
+
+		ControlPc = context.Rip;
+		// Check if we left the user range.
+		if (ControlPc < 0x10000) {
+			break;
+		}
+
+		BackTrace[i] = (PVOID)(ControlPc);
+	}
+	return i;
+#else
+	return 0;
+#endif
+}
+*/
+import "C"
+
+import (
+	"internal/testenv"
+	"reflect"
+	"runtime"
+	"strings"
+	"testing"
+	"unsafe"
+)
+
+// Test that the stack can be unwound through a call out and call back
+// into Go.
+func testCallbackCallersSEH(t *testing.T) {
+	testenv.SkipIfOptimizationOff(t) // This test requires inlining.
+	if runtime.Compiler != "gc" {
+		// The exact function names are not going to be the same.
+		t.Skip("skipping for non-gc toolchain")
+	}
+	if runtime.GOARCH != "amd64" {
+		// TODO: support SEH on other architectures.
+		t.Skip("skipping on non-amd64")
+	}
+	const cgoexpPrefix = "_cgoexp_"
+	want := []string{
+		"runtime.asmcgocall_landingpad",
+		"runtime.asmcgocall",
+		"runtime.cgocall",
+		"test._Cfunc_backtrace",
+		"test.testCallbackCallersSEH.func1.1",
+		"test.testCallbackCallersSEH.func1",
+		"test.goCallback",
+		cgoexpPrefix + "goCallback",
+		"runtime.cgocallbackg1",
+		"runtime.cgocallbackg",
+		"runtime.cgocallbackg",
+		"runtime.cgocallback",
+		"crosscall2",
+		"runtime.asmcgocall_landingpad",
+		"runtime.asmcgocall",
+		"runtime.cgocall",
+		"test._Cfunc_callback",
+		"test.nestedCall.func1",
+		"test.nestedCall",
+		"test.testCallbackCallersSEH",
+		"test.TestCallbackCallersSEH",
+		"testing.tRunner",
+		"testing.(*T).Run.gowrap3",
+		"runtime.goexit",
+	}
+	pc := make([]uintptr, 100)
+	n := 0
+	nestedCall(func() {
+		n = int(C.backtrace(C.DWORD(len(pc)), (*C.PVOID)(unsafe.Pointer(&pc[0]))))
+	})
+	got := make([]string, 0, n)
+	for i := 0; i < n; i++ {
+		f := runtime.FuncForPC(pc[i] - 1)
+		if f == nil {
+			continue
+		}
+		fname := f.Name()
+		switch fname {
+		case "goCallback", "callback":
+			// TODO(qmuntal): investigate why these functions don't appear
+			// when using the external linker.
+			continue
+		}
+		// Skip cgo-generated functions, the runtime might not know about them,
+		// depending on the link mode.
+		if strings.HasPrefix(fname, "_cgo_") {
+			continue
+		}
+		// Remove the cgo-generated random prefix.
+		if strings.HasPrefix(fname, cgoexpPrefix) {
+			idx := strings.Index(fname[len(cgoexpPrefix):], "_")
+			if idx >= 0 {
+				fname = cgoexpPrefix + fname[len(cgoexpPrefix)+idx+1:]
+			}
+		}
+		// In module mode, this package has a fully-qualified import path.
+		// Remove it if present.
+		fname = strings.TrimPrefix(fname, "cmd/cgo/internal/")
+		got = append(got, fname)
+	}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("incorrect backtrace:\nwant:\t%v\ngot:\t%v", want, got)
+	}
+}

src/cmd/cgo/internal/test/cgo_windows_test.go

@@ -0,0 +1,11 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build cgo && windows
+
+package cgotest
+
+import "testing"
+
+func TestCallbackCallersSEH(t *testing.T) { testCallbackCallersSEH(t) }

src/cmd/cgo/internal/test/issue4339.c

@@ -1,3 +1,7 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 #include <stdio.h>
 #include "issue4339.h"
 

src/cmd/cgo/internal/test/issue4339.h

@@ -1,3 +1,7 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 typedef struct Issue4339 Issue4339;
 
 struct Issue4339 {

src/cmd/cgo/internal/test/issue8756.go

@@ -1,3 +1,7 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package cgotest
 
 /*

src/cmd/cgo/internal/test/issue8756/issue8756.go

@@ -1,3 +1,7 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package issue8756
 
 /*

src/cmd/cgo/internal/test/issue8828/trivial.go

@@ -1,3 +1,7 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package issue8828
 
 //void foo();

src/cmd/cgo/internal/test/issue9026/issue9026.go

@@ -1,3 +1,7 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package issue9026
 
 // This file appears in its own package since the assertion tests the

src/cmd/cgo/internal/test/issue9510a/a.go

@@ -1,3 +1,7 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package issue9510a
 
 /*

src/cmd/cgo/internal/test/issue9510b/b.go

@@ -1,3 +1,7 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package issue9510b
 
 /*

src/cmd/compile/abi-internal.md

@@ -633,6 +633,56 @@ modifying or saving the FPCR.
 Functions are allowed to modify it between calls (as long as they
 restore it), but as of this writing Go code never does.
 
+### loong64 architecture
+
+The loong64 architecture uses R4 – R19 for integer arguments and integer results.
+
+It uses F0 – F15 for floating-point arguments and results.
+
+Registers R20 - R21, R23 – R28, R30 - R31, F16 – F31 are permanent scratch registers.
+
+Register R2 is reserved and never used.
+
+Register R20, R21 is Used by runtime.duffcopy, runtime.duffzero.
+
+Special-purpose registers used within Go generated code and Go assembly code
+are as follows:
+
+| Register | Call meaning | Return meaning | Body meaning |
+| --- | --- | --- | --- |
+| R0 | Zero value | Same | Same |
+| R1 | Link register | Link register | Scratch |
+| R3 | Stack pointer | Same | Same |
+| R20,R21 | Scratch | Scratch | Used by duffcopy, duffzero |
+| R22 | Current goroutine | Same | Same |
+| R29 | Closure context pointer | Same | Same |
+| R30, R31 | used by the assembler | Same | Same |
+
+*Rationale*: These register meanings are compatible with Go’s stack-based
+calling convention.
+
+#### Stack layout
+
+The stack pointer, R3, grows down and is aligned to 8 bytes.
+
+A function's stack frame, after the frame is created, is laid out as
+follows:
+
+    +------------------------------+
+    | ... locals ...               |
+    | ... outgoing arguments ...   |
+    | return PC                    | ← R3 points to
+    +------------------------------+ ↓ lower addresses
+
+This stack layout is used by both register-based (ABIInternal) and
+stack-based (ABI0) calling conventions.
+
+The "return PC" is loaded to the link register, R1, as part of the
+loong64 `JAL` operation.
+
+#### Flags
+All bits in CSR are system flags and are not modified by Go.
+
 ### ppc64 architecture
 
 The ppc64 architecture uses R3 – R10 and R14 – R17 for integer arguments

src/cmd/compile/internal/arm/galign.go

@@ -15,7 +15,7 @@ func Init(arch *ssagen.ArchInfo) {
 	arch.LinkArch = &arm.Linkarm
 	arch.REGSP = arm.REGSP
 	arch.MAXWIDTH = (1 << 32) - 1
-	arch.SoftFloat = buildcfg.GOARM == 5
+	arch.SoftFloat = buildcfg.GOARM.SoftFloat
 	arch.ZeroRange = zerorange
 	arch.Ginsnop = ginsnop
 

src/cmd/compile/internal/arm/ssa.go

@@ -289,7 +289,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 	case ssa.OpARMANDconst, ssa.OpARMBICconst:
 		// try to optimize ANDconst and BICconst to BFC, which saves bytes and ticks
 		// BFC is only available on ARMv7, and its result and source are in the same register
-		if buildcfg.GOARM == 7 && v.Reg() == v.Args[0].Reg() {
+		if buildcfg.GOARM.Version == 7 && v.Reg() == v.Args[0].Reg() {
 			var val uint32
 			if v.Op == ssa.OpARMANDconst {
 				val = ^uint32(v.AuxInt)
@@ -646,7 +646,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 			default:
 			}
 		}
-		if buildcfg.GOARM >= 6 {
+		if buildcfg.GOARM.Version >= 6 {
 			// generate more efficient "MOVB/MOVBU/MOVH/MOVHU Reg@>0, Reg" on ARMv6 & ARMv7
 			genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm.SHIFT_RR, 0)
 			return

src/cmd/compile/internal/base/debug.go

@@ -23,6 +23,8 @@ type DebugFlags struct {
 	DisableNil            int    `help:"disable nil checks" concurrent:"ok"`
 	DumpInlFuncProps      string `help:"dump function properties from inl heuristics to specified file"`
 	DumpInlCallSiteScores int    `help:"dump scored callsites during inlining"`
+	InlScoreAdj           string `help:"set inliner score adjustments (ex: -d=inlscoreadj=panicPathAdj:10/passConstToNestedIfAdj:-90)"`
+	InlBudgetSlack        int    `help:"amount to expand the initial inline budget when new inliner enabled. Defaults to 80 if option not set." concurrent:"ok"`
 	DumpPtrs              int    `help:"show Node pointers values in dump output"`
 	DwarfInl              int    `help:"print information about DWARF inlined function creation"`
 	EscapeMutationsCalls  int    `help:"print extra escape analysis diagnostics about mutations and calls" concurrent:"ok"`
@@ -59,7 +61,8 @@ type DebugFlags struct {
 	PGOInline             int    `help:"enable profile-guided inlining" concurrent:"ok"`
 	PGOInlineCDFThreshold string `help:"cumulative threshold percentage for determining call sites as hot candidates for inlining" concurrent:"ok"`
 	PGOInlineBudget       int    `help:"inline budget for hot functions" concurrent:"ok"`
-	PGODevirtualize       int    `help:"enable profile-guided devirtualization" concurrent:"ok"`
+	PGODevirtualize       int    `help:"enable profile-guided devirtualization; 0 to disable, 1 to enable interface devirtualization, 2 to enable function devirtualization" concurrent:"ok"`
+	RangeFuncCheck        int    `help:"insert code to check behavior of range iterator functions" concurrent:"ok"`
 	WrapGlobalMapDbg      int    `help:"debug trace output for global map init wrapping"`
 	WrapGlobalMapCtl      int    `help:"global map init wrap control (0 => default, 1 => off, 2 => stress mode, no size cutoff)"`
 	ZeroCopy              int    `help:"enable zero-copy string->[]byte conversions" concurrent:"ok"`

src/cmd/compile/internal/base/flag.go

@@ -179,9 +179,10 @@ func ParseFlags() {
 	Debug.InlFuncsWithClosures = 1
 	Debug.InlStaticInit = 1
 	Debug.PGOInline = 1
-	Debug.PGODevirtualize = 1
+	Debug.PGODevirtualize = 2
 	Debug.SyncFrames = -1 // disable sync markers by default
 	Debug.ZeroCopy = 1
+	Debug.RangeFuncCheck = 1
 
 	Debug.Checkptr = -1 // so we can tell whether it is set explicitly
 

src/cmd/compile/internal/devirtualize/devirtualize.go

@@ -18,39 +18,27 @@ import (
 	"cmd/compile/internal/types"
 )
 
-// Static devirtualizes calls within fn where possible when the concrete callee
+// StaticCall devirtualizes the given call if possible when the concrete callee
 // is available statically.
-func Static(fn *ir.Func) {
-	ir.CurFunc = fn
+func StaticCall(call *ir.CallExpr) {
+	// For promoted methods (including value-receiver methods promoted
+	// to pointer-receivers), the interface method wrapper may contain
+	// expressions that can panic (e.g., ODEREF, ODOTPTR,
+	// ODOTINTER). Devirtualization involves inlining these expressions
+	// (and possible panics) to the call site. This normally isn't a
+	// problem, but for go/defer statements it can move the panic from
+	// when/where the call executes to the go/defer statement itself,
+	// which is a visible change in semantics (e.g., #52072). To prevent
+	// this, we skip devirtualizing calls within go/defer statements
+	// altogether.
+	if call.GoDefer {
+		return
+	}
 
-	// For promoted methods (including value-receiver methods promoted to pointer-receivers),
-	// the interface method wrapper may contain expressions that can panic (e.g., ODEREF, ODOTPTR, ODOTINTER).
-	// Devirtualization involves inlining these expressions (and possible panics) to the call site.
-	// This normally isn't a problem, but for go/defer statements it can move the panic from when/where
-	// the call executes to the go/defer statement itself, which is a visible change in semantics (e.g., #52072).
-	// To prevent this, we skip devirtualizing calls within go/defer statements altogether.
-	goDeferCall := make(map[*ir.CallExpr]bool)
-	ir.VisitList(fn.Body, func(n ir.Node) {
-		switch n := n.(type) {
-		case *ir.GoDeferStmt:
-			if call, ok := n.Call.(*ir.CallExpr); ok {
-				goDeferCall[call] = true
-			}
-			return
-		case *ir.CallExpr:
-			if !goDeferCall[n] {
-				staticCall(n)
-			}
-		}
-	})
-}
-
-// staticCall devirtualizes the given call if possible when the concrete callee
-// is available statically.
-func staticCall(call *ir.CallExpr) {
 	if call.Op() != ir.OCALLINTER {
 		return
 	}
+
 	sel := call.Fun.(*ir.SelectorExpr)
 	r := ir.StaticValue(sel.X)
 	if r.Op() != ir.OCONVIFACE {
@@ -70,7 +58,7 @@ func staticCall(call *ir.CallExpr) {
 		return
 	}
 
-	// If typ *has* a shape type, then it's an shaped, instantiated
+	// If typ *has* a shape type, then it's a shaped, instantiated
 	// type like T[go.shape.int], and its methods (may) have an extra
 	// dictionary parameter. We could devirtualize this call if we
 	// could derive an appropriate dictionary argument.

src/cmd/compile/internal/devirtualize/pgo.go

@@ -12,6 +12,8 @@ import (
 	"cmd/compile/internal/pgo"
 	"cmd/compile/internal/typecheck"
 	"cmd/compile/internal/types"
+	"cmd/internal/obj"
+	"cmd/internal/src"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -53,8 +55,10 @@ type CallStat struct {
 // ProfileGuided performs call devirtualization of indirect calls based on
 // profile information.
 //
-// Specifically, it performs conditional devirtualization of interface calls
-// for the hottest callee. That is, it performs a transformation like:
+// Specifically, it performs conditional devirtualization of interface calls or
+// function value calls for the hottest callee.
+//
+// That is, for interface calls it performs a transformation like:
 //
 //	type Iface interface {
 //		Foo()
@@ -78,6 +82,24 @@ type CallStat struct {
 //		}
 //	}
 //
+// For function value calls it performs a transformation like:
+//
+//	func Concrete() {}
+//
+//	func foo(fn func()) {
+//		fn()
+//	}
+//
+// to:
+//
+//	func foo(fn func()) {
+//		if internal/abi.FuncPCABIInternal(fn) == internal/abi.FuncPCABIInternal(Concrete) {
+//			Concrete()
+//		} else {
+//			fn()
+//		}
+//	}
+//
 // The primary benefit of this transformation is enabling inlining of the
 // direct call.
 func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
@@ -85,9 +107,6 @@ func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
 
 	name := ir.LinkFuncName(fn)
 
-	// Can't devirtualize go/defer calls. See comment in Static.
-	goDeferCall := make(map[*ir.CallExpr]bool)
-
 	var jsonW *json.Encoder
 	if base.Debug.PGODebug >= 3 {
 		jsonW = json.NewEncoder(os.Stdout)
@@ -99,12 +118,6 @@ func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
 			return n
 		}
 
-		if gds, ok := n.(*ir.GoDeferStmt); ok {
-			if call, ok := gds.Call.(*ir.CallExpr); ok {
-				goDeferCall[call] = true
-			}
-		}
-
 		ir.EditChildren(n, edit)
 
 		call, ok := n.(*ir.CallExpr)
@@ -125,7 +138,8 @@ func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
 			}
 		}
 
-		if call.Op() != ir.OCALLINTER {
+		op := call.Op()
+		if op != ir.OCALLFUNC && op != ir.OCALLINTER {
 			return n
 		}
 
@@ -133,30 +147,26 @@ func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
 			fmt.Printf("%v: PGO devirtualize considering call %v\n", ir.Line(call), call)
 		}
 
-		if goDeferCall[call] {
+		if call.GoDefer {
 			if base.Debug.PGODebug >= 2 {
 				fmt.Printf("%v: can't PGO devirtualize go/defer call %v\n", ir.Line(call), call)
 			}
 			return n
 		}
 
-		// Bail if we do not have a hot callee.
-		callee, weight := findHotConcreteCallee(p, fn, call)
-		if callee == nil {
-			return n
-		}
-		// Bail if we do not have a Type node for the hot callee.
-		ctyp := methodRecvType(callee)
-		if ctyp == nil {
-			return n
-		}
-		// Bail if we know for sure it won't inline.
-		if !shouldPGODevirt(callee) {
-			return n
+		var newNode ir.Node
+		var callee *ir.Func
+		var weight int64
+		switch op {
+		case ir.OCALLFUNC:
+			newNode, callee, weight = maybeDevirtualizeFunctionCall(p, fn, call)
+		case ir.OCALLINTER:
+			newNode, callee, weight = maybeDevirtualizeInterfaceCall(p, fn, call)
+		default:
+			panic("unreachable")
 		}
 
-		if !base.PGOHash.MatchPosWithInfo(n.Pos(), "devirt", nil) {
-			// De-selected by PGO Hash.
+		if newNode == nil {
 			return n
 		}
 
@@ -165,12 +175,126 @@ func ProfileGuided(fn *ir.Func, p *pgo.Profile) {
 			stat.DevirtualizedWeight = weight
 		}
 
-		return rewriteCondCall(call, fn, callee, ctyp)
+		return newNode
 	}
 
 	ir.EditChildren(fn, edit)
 }
 
+// Devirtualize interface call if possible and eligible. Returns the new
+// ir.Node if call was devirtualized, and if so also the callee and weight of
+// the devirtualized edge.
+func maybeDevirtualizeInterfaceCall(p *pgo.Profile, fn *ir.Func, call *ir.CallExpr) (ir.Node, *ir.Func, int64) {
+	if base.Debug.PGODevirtualize < 1 {
+		return nil, nil, 0
+	}
+
+	// Bail if we do not have a hot callee.
+	callee, weight := findHotConcreteInterfaceCallee(p, fn, call)
+	if callee == nil {
+		return nil, nil, 0
+	}
+	// Bail if we do not have a Type node for the hot callee.
+	ctyp := methodRecvType(callee)
+	if ctyp == nil {
+		return nil, nil, 0
+	}
+	// Bail if we know for sure it won't inline.
+	if !shouldPGODevirt(callee) {
+		return nil, nil, 0
+	}
+	// Bail if de-selected by PGO Hash.
+	if !base.PGOHash.MatchPosWithInfo(call.Pos(), "devirt", nil) {
+		return nil, nil, 0
+	}
+
+	return rewriteInterfaceCall(call, fn, callee, ctyp), callee, weight
+}
+
+// Devirtualize an indirect function call if possible and eligible. Returns the new
+// ir.Node if call was devirtualized, and if so also the callee and weight of
+// the devirtualized edge.
+func maybeDevirtualizeFunctionCall(p *pgo.Profile, fn *ir.Func, call *ir.CallExpr) (ir.Node, *ir.Func, int64) {
+	if base.Debug.PGODevirtualize < 2 {
+		return nil, nil, 0
+	}
+
+	// Bail if this is a direct call; no devirtualization necessary.
+	callee := pgo.DirectCallee(call.Fun)
+	if callee != nil {
+		return nil, nil, 0
+	}
+
+	// Bail if we do not have a hot callee.
+	callee, weight := findHotConcreteFunctionCallee(p, fn, call)
+	if callee == nil {
+		return nil, nil, 0
+	}
+
+	// TODO(go.dev/issue/61577): Closures need the closure context passed
+	// via the context register. That requires extra plumbing that we
+	// haven't done yet.
+	if callee.OClosure != nil {
+		if base.Debug.PGODebug >= 3 {
+			fmt.Printf("callee %s is a closure, skipping\n", ir.FuncName(callee))
+		}
+		return nil, nil, 0
+	}
+	// runtime.memhash_varlen does not look like a closure, but it uses
+	// runtime.getclosureptr to access data encoded by callers, which are
+	// are generated by cmd/compile/internal/reflectdata.genhash.
+	if callee.Sym().Pkg.Path == "runtime" && callee.Sym().Name == "memhash_varlen" {
+		if base.Debug.PGODebug >= 3 {
+			fmt.Printf("callee %s is a closure (runtime.memhash_varlen), skipping\n", ir.FuncName(callee))
+		}
+		return nil, nil, 0
+	}
+	// TODO(prattmic): We don't properly handle methods as callees in two
+	// different dimensions:
+	//
+	// 1. Method expressions. e.g.,
+	//
+	//      var fn func(*os.File, []byte) (int, error) = (*os.File).Read
+	//
+	// In this case, typ will report *os.File as the receiver while
+	// ctyp reports it as the first argument. types.Identical ignores
+	// receiver parameters, so it treats these as different, even though
+	// they are still call compatible.
+	//
+	// 2. Method values. e.g.,
+	//
+	//      var f *os.File
+	//      var fn func([]byte) (int, error) = f.Read
+	//
+	// types.Identical will treat these as compatible (since receiver
+	// parameters are ignored). However, in this case, we do not call
+	// (*os.File).Read directly. Instead, f is stored in closure context
+	// and we call the wrapper (*os.File).Read-fm. However, runtime/pprof
+	// hides wrappers from profiles, making it appear that there is a call
+	// directly to the method. We could recognize this pattern return the
+	// wrapper rather than the method.
+	//
+	// N.B. perf profiles will report wrapper symbols directly, so
+	// ideally we should support direct wrapper references as well.
+	if callee.Type().Recv() != nil {
+		if base.Debug.PGODebug >= 3 {
+			fmt.Printf("callee %s is a method, skipping\n", ir.FuncName(callee))
+		}
+		return nil, nil, 0
+	}
+
+	// Bail if we know for sure it won't inline.
+	if !shouldPGODevirt(callee) {
+		return nil, nil, 0
+	}
+	// Bail if de-selected by PGO Hash.
+	if !base.PGOHash.MatchPosWithInfo(call.Pos(), "devirt", nil) {
+		return nil, nil, 0
+	}
+
+	return rewriteFunctionCall(call, fn, callee), callee, weight
+}
+
 // shouldPGODevirt checks if we should perform PGO devirtualization to the
 // target function.
 //
@@ -279,11 +403,90 @@ func constructCallStat(p *pgo.Profile, fn *ir.Func, name string, call *ir.CallEx
 	return &stat
 }
 
-// rewriteCondCall devirtualizes the given call using a direct method call to
-// concretetyp.
-func rewriteCondCall(call *ir.CallExpr, curfn, callee *ir.Func, concretetyp *types.Type) ir.Node {
+// copyInputs copies the inputs to a call: the receiver (for interface calls)
+// or function value (for function value calls) and the arguments. These
+// expressions are evaluated once and assigned to temporaries.
+//
+// The assignment statement is added to init and the copied receiver/fn
+// expression and copied arguments expressions are returned.
+func copyInputs(curfn *ir.Func, pos src.XPos, recvOrFn ir.Node, args []ir.Node, init *ir.Nodes) (ir.Node, []ir.Node) {
+	// Evaluate receiver/fn and argument expressions. The receiver/fn is
+	// used twice but we don't want to cause side effects twice. The
+	// arguments are used in two different calls and we can't trivially
+	// copy them.
+	//
+	// recvOrFn must be first in the assignment list as its side effects
+	// must be ordered before argument side effects.
+	var lhs, rhs []ir.Node
+	newRecvOrFn := typecheck.TempAt(pos, curfn, recvOrFn.Type())
+	lhs = append(lhs, newRecvOrFn)
+	rhs = append(rhs, recvOrFn)
+
+	for _, arg := range args {
+		argvar := typecheck.TempAt(pos, curfn, arg.Type())
+
+		lhs = append(lhs, argvar)
+		rhs = append(rhs, arg)
+	}
+
+	asList := ir.NewAssignListStmt(pos, ir.OAS2, lhs, rhs)
+	init.Append(typecheck.Stmt(asList))
+
+	return newRecvOrFn, lhs[1:]
+}
+
+// retTemps returns a slice of temporaries to be used for storing result values from call.
+func retTemps(curfn *ir.Func, pos src.XPos, call *ir.CallExpr) []ir.Node {
+	sig := call.Fun.Type()
+	var retvars []ir.Node
+	for _, ret := range sig.Results() {
+		retvars = append(retvars, typecheck.TempAt(pos, curfn, ret.Type))
+	}
+	return retvars
+}
+
+// condCall returns an ir.InlinedCallExpr that performs a call to thenCall if
+// cond is true and elseCall if cond is false. The return variables of the
+// InlinedCallExpr evaluate to the return values from the call.
+func condCall(curfn *ir.Func, pos src.XPos, cond ir.Node, thenCall, elseCall *ir.CallExpr, init ir.Nodes) *ir.InlinedCallExpr {
+	// Doesn't matter whether we use thenCall or elseCall, they must have
+	// the same return types.
+	retvars := retTemps(curfn, pos, thenCall)
+
+	var thenBlock, elseBlock ir.Nodes
+	if len(retvars) == 0 {
+		thenBlock.Append(thenCall)
+		elseBlock.Append(elseCall)
+	} else {
+		// Copy slice so edits in one location don't affect another.
+		thenRet := append([]ir.Node(nil), retvars...)
+		thenAsList := ir.NewAssignListStmt(pos, ir.OAS2, thenRet, []ir.Node{thenCall})
+		thenBlock.Append(typecheck.Stmt(thenAsList))
+
+		elseRet := append([]ir.Node(nil), retvars...)
+		elseAsList := ir.NewAssignListStmt(pos, ir.OAS2, elseRet, []ir.Node{elseCall})
+		elseBlock.Append(typecheck.Stmt(elseAsList))
+	}
+
+	nif := ir.NewIfStmt(pos, cond, thenBlock, elseBlock)
+	nif.SetInit(init)
+	nif.Likely = true
+
+	body := []ir.Node{typecheck.Stmt(nif)}
+
+	// This isn't really an inlined call of course, but InlinedCallExpr
+	// makes handling reassignment of return values easier.
+	res := ir.NewInlinedCallExpr(pos, body, retvars)
+	res.SetType(thenCall.Type())
+	res.SetTypecheck(1)
+	return res
+}
+
+// rewriteInterfaceCall devirtualizes the given interface call using a direct
+// method call to concretetyp.
+func rewriteInterfaceCall(call *ir.CallExpr, curfn, callee *ir.Func, concretetyp *types.Type) ir.Node {
 	if base.Flag.LowerM != 0 {
-		fmt.Printf("%v: PGO devirtualizing %v to %v\n", ir.Line(call), call.Fun, callee)
+		fmt.Printf("%v: PGO devirtualizing interface call %v to %v\n", ir.Line(call), call.Fun, callee)
 	}
 
 	// We generate an OINCALL of:
@@ -314,46 +517,15 @@ func rewriteCondCall(call *ir.CallExpr, curfn, callee *ir.Func, concretetyp *typ
 	// making it less like to inline. We may want to compensate for this
 	// somehow.
 
-	var retvars []ir.Node
-
-	sig := call.Fun.Type()
-
-	for _, ret := range sig.Results() {
-		retvars = append(retvars, typecheck.TempAt(base.Pos, curfn, ret.Type))
-	}
-
 	sel := call.Fun.(*ir.SelectorExpr)
 	method := sel.Sel
 	pos := call.Pos()
 	init := ir.TakeInit(call)
 
-	// Evaluate receiver and argument expressions. The receiver is used
-	// twice but we don't want to cause side effects twice. The arguments
-	// are used in two different calls and we can't trivially copy them.
-	//
-	// recv must be first in the assignment list as its side effects must
-	// be ordered before argument side effects.
-	var lhs, rhs []ir.Node
-	recv := typecheck.TempAt(base.Pos, curfn, sel.X.Type())
-	lhs = append(lhs, recv)
-	rhs = append(rhs, sel.X)
-
-	// Move arguments to assignments prior to the if statement. We cannot
-	// simply copy the args' IR, as some IR constructs cannot be copied,
-	// such as labels (possible in InlinedCall nodes).
-	args := call.Args.Take()
-	for _, arg := range args {
-		argvar := typecheck.TempAt(base.Pos, curfn, arg.Type())
-
-		lhs = append(lhs, argvar)
-		rhs = append(rhs, arg)
-	}
-
-	asList := ir.NewAssignListStmt(pos, ir.OAS2, lhs, rhs)
-	init.Append(typecheck.Stmt(asList))
+	recv, args := copyInputs(curfn, pos, sel.X, call.Args.Take(), &init)
 
 	// Copy slice so edits in one location don't affect another.
-	argvars := append([]ir.Node(nil), lhs[1:]...)
+	argvars := append([]ir.Node(nil), args...)
 	call.Args = argvars
 
 	tmpnode := typecheck.TempAt(base.Pos, curfn, concretetyp)
@@ -367,38 +539,84 @@ func rewriteCondCall(call *ir.CallExpr, curfn, callee *ir.Func, concretetyp *typ
 	concreteCallee := typecheck.XDotMethod(pos, tmpnode, method, true)
 	// Copy slice so edits in one location don't affect another.
 	argvars = append([]ir.Node(nil), argvars...)
-	concreteCall := typecheck.Call(pos, concreteCallee, argvars, call.IsDDD)
+	concreteCall := typecheck.Call(pos, concreteCallee, argvars, call.IsDDD).(*ir.CallExpr)
 
-	var thenBlock, elseBlock ir.Nodes
-	if len(retvars) == 0 {
-		thenBlock.Append(concreteCall)
-		elseBlock.Append(call)
-	} else {
-		// Copy slice so edits in one location don't affect another.
-		thenRet := append([]ir.Node(nil), retvars...)
-		thenAsList := ir.NewAssignListStmt(pos, ir.OAS2, thenRet, []ir.Node{concreteCall})
-		thenBlock.Append(typecheck.Stmt(thenAsList))
-
-		elseRet := append([]ir.Node(nil), retvars...)
-		elseAsList := ir.NewAssignListStmt(pos, ir.OAS2, elseRet, []ir.Node{call})
-		elseBlock.Append(typecheck.Stmt(elseAsList))
-	}
-
-	cond := ir.NewIfStmt(pos, nil, nil, nil)
-	cond.SetInit(init)
-	cond.Cond = tmpok
-	cond.Body = thenBlock
-	cond.Else = elseBlock
-	cond.Likely = true
-
-	body := []ir.Node{typecheck.Stmt(cond)}
-
-	res := ir.NewInlinedCallExpr(pos, body, retvars)
-	res.SetType(call.Type())
-	res.SetTypecheck(1)
+	res := condCall(curfn, pos, tmpok, concreteCall, call, init)
 
 	if base.Debug.PGODebug >= 3 {
-		fmt.Printf("PGO devirtualizing call to %+v. After: %+v\n", concretetyp, res)
+		fmt.Printf("PGO devirtualizing interface call to %+v. After: %+v\n", concretetyp, res)
+	}
+
+	return res
+}
+
+// rewriteFunctionCall devirtualizes the given OCALLFUNC using a direct
+// function call to callee.
+func rewriteFunctionCall(call *ir.CallExpr, curfn, callee *ir.Func) ir.Node {
+	if base.Flag.LowerM != 0 {
+		fmt.Printf("%v: PGO devirtualizing function call %v to %v\n", ir.Line(call), call.Fun, callee)
+	}
+
+	// We generate an OINCALL of:
+	//
+	// var fn FuncType
+	//
+	// var arg1 A1
+	// var argN AN
+	//
+	// var ret1 R1
+	// var retN RN
+	//
+	// fn, arg1, argN = fn expr, arg1 expr, argN expr
+	//
+	// fnPC := internal/abi.FuncPCABIInternal(fn)
+	// concretePC := internal/abi.FuncPCABIInternal(concrete)
+	//
+	// if fnPC == concretePC {
+	//   ret1, retN = concrete(arg1, ... argN) // Same closure context passed (TODO)
+	// } else {
+	//   ret1, retN = fn(arg1, ... argN)
+	// }
+	//
+	// OINCALL retvars: ret1, ... retN
+	//
+	// This isn't really an inlined call of course, but InlinedCallExpr
+	// makes handling reassignment of return values easier.
+
+	pos := call.Pos()
+	init := ir.TakeInit(call)
+
+	fn, args := copyInputs(curfn, pos, call.Fun, call.Args.Take(), &init)
+
+	// Copy slice so edits in one location don't affect another.
+	argvars := append([]ir.Node(nil), args...)
+	call.Args = argvars
+
+	// FuncPCABIInternal takes an interface{}, emulate that. This is needed
+	// for to ensure we get the MAKEFACE we need for SSA.
+	fnIface := typecheck.Expr(ir.NewConvExpr(pos, ir.OCONV, types.Types[types.TINTER], fn))
+	calleeIface := typecheck.Expr(ir.NewConvExpr(pos, ir.OCONV, types.Types[types.TINTER], callee.Nname))
+
+	fnPC := ir.FuncPC(pos, fnIface, obj.ABIInternal)
+	concretePC := ir.FuncPC(pos, calleeIface, obj.ABIInternal)
+
+	pcEq := typecheck.Expr(ir.NewBinaryExpr(base.Pos, ir.OEQ, fnPC, concretePC))
+
+	// TODO(go.dev/issue/61577): Handle callees that a closures and need a
+	// copy of the closure context from call. For now, we skip callees that
+	// are closures in maybeDevirtualizeFunctionCall.
+	if callee.OClosure != nil {
+		base.Fatalf("Callee is a closure: %+v", callee)
+	}
+
+	// Copy slice so edits in one location don't affect another.
+	argvars = append([]ir.Node(nil), argvars...)
+	concreteCall := typecheck.Call(pos, callee.Nname, argvars, call.IsDDD).(*ir.CallExpr)
+
+	res := condCall(curfn, pos, pcEq, concreteCall, call, init)
+
+	if base.Debug.PGODebug >= 3 {
+		fmt.Printf("PGO devirtualizing function call to %+v. After: %+v\n", ir.FuncName(callee), res)
 	}
 
 	return res
@@ -429,15 +647,15 @@ func interfaceCallRecvTypeAndMethod(call *ir.CallExpr) (*types.Type, *types.Sym)
 	return sel.X.Type(), sel.Sel
 }
 
-// findHotConcreteCallee returns the *ir.Func of the hottest callee of an
-// indirect call, if available, and its edge weight.
-func findHotConcreteCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr) (*ir.Func, int64) {
+// findHotConcreteCallee returns the *ir.Func of the hottest callee of a call,
+// if available, and its edge weight. extraFn can perform additional
+// applicability checks on each candidate edge. If extraFn returns false,
+// candidate will not be considered a valid callee candidate.
+func findHotConcreteCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr, extraFn func(callerName string, callOffset int, candidate *pgo.IREdge) bool) (*ir.Func, int64) {
 	callerName := ir.LinkFuncName(caller)
 	callerNode := p.WeightedCG.IRNodes[callerName]
 	callOffset := pgo.NodeLineOffset(call, caller)
 
-	inter, method := interfaceCallRecvTypeAndMethod(call)
-
 	var hottest *pgo.IREdge
 
 	// Returns true if e is hotter than hottest.
@@ -504,41 +722,7 @@ func findHotConcreteCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr) (
 			continue
 		}
 
-		ctyp := methodRecvType(e.Dst.AST)
-		if ctyp == nil {
-			// Not a method.
-			// TODO(prattmic): Support non-interface indirect calls.
-			if base.Debug.PGODebug >= 2 {
-				fmt.Printf("%v: edge %s:%d -> %s (weight %d): callee not a method\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight)
-			}
-			continue
-		}
-
-		// If ctyp doesn't implement inter it is most likely from a
-		// different call on the same line
-		if !typecheck.Implements(ctyp, inter) {
-			// TODO(prattmic): this is overly strict. Consider if
-			// ctyp is a partial implementation of an interface
-			// that gets embedded in types that complete the
-			// interface. It would still be OK to devirtualize a
-			// call to this method.
-			//
-			// What we'd need to do is check that the function
-			// pointer in the itab matches the method we want,
-			// rather than doing a full type assertion.
-			if base.Debug.PGODebug >= 2 {
-				why := typecheck.ImplementsExplain(ctyp, inter)
-				fmt.Printf("%v: edge %s:%d -> %s (weight %d): %v doesn't implement %v (%s)\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight, ctyp, inter, why)
-			}
-			continue
-		}
-
-		// If the method name is different it is most likely from a
-		// different call on the same line
-		if !strings.HasSuffix(e.Dst.Name(), "."+method.Name) {
-			if base.Debug.PGODebug >= 2 {
-				fmt.Printf("%v: edge %s:%d -> %s (weight %d): callee is a different method\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight)
-			}
+		if extraFn != nil && !extraFn(callerName, callOffset, e) {
 			continue
 		}
 
@@ -560,3 +744,77 @@ func findHotConcreteCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr) (
 	}
 	return hottest.Dst.AST, hottest.Weight
 }
+
+// findHotConcreteInterfaceCallee returns the *ir.Func of the hottest callee of an
+// interface call, if available, and its edge weight.
+func findHotConcreteInterfaceCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr) (*ir.Func, int64) {
+	inter, method := interfaceCallRecvTypeAndMethod(call)
+
+	return findHotConcreteCallee(p, caller, call, func(callerName string, callOffset int, e *pgo.IREdge) bool {
+		ctyp := methodRecvType(e.Dst.AST)
+		if ctyp == nil {
+			// Not a method.
+			// TODO(prattmic): Support non-interface indirect calls.
+			if base.Debug.PGODebug >= 2 {
+				fmt.Printf("%v: edge %s:%d -> %s (weight %d): callee not a method\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight)
+			}
+			return false
+		}
+
+		// If ctyp doesn't implement inter it is most likely from a
+		// different call on the same line
+		if !typecheck.Implements(ctyp, inter) {
+			// TODO(prattmic): this is overly strict. Consider if
+			// ctyp is a partial implementation of an interface
+			// that gets embedded in types that complete the
+			// interface. It would still be OK to devirtualize a
+			// call to this method.
+			//
+			// What we'd need to do is check that the function
+			// pointer in the itab matches the method we want,
+			// rather than doing a full type assertion.
+			if base.Debug.PGODebug >= 2 {
+				why := typecheck.ImplementsExplain(ctyp, inter)
+				fmt.Printf("%v: edge %s:%d -> %s (weight %d): %v doesn't implement %v (%s)\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight, ctyp, inter, why)
+			}
+			return false
+		}
+
+		// If the method name is different it is most likely from a
+		// different call on the same line
+		if !strings.HasSuffix(e.Dst.Name(), "."+method.Name) {
+			if base.Debug.PGODebug >= 2 {
+				fmt.Printf("%v: edge %s:%d -> %s (weight %d): callee is a different method\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight)
+			}
+			return false
+		}
+
+		return true
+	})
+}
+
+// findHotConcreteFunctionCallee returns the *ir.Func of the hottest callee of an
+// indirect function call, if available, and its edge weight.
+func findHotConcreteFunctionCallee(p *pgo.Profile, caller *ir.Func, call *ir.CallExpr) (*ir.Func, int64) {
+	typ := call.Fun.Type().Underlying()
+
+	return findHotConcreteCallee(p, caller, call, func(callerName string, callOffset int, e *pgo.IREdge) bool {
+		ctyp := e.Dst.AST.Type().Underlying()
+
+		// If ctyp doesn't match typ it is most likely from a different
+		// call on the same line.
+		//
+		// Note that we are comparing underlying types, as different
+		// defined types are OK. e.g., a call to a value of type
+		// net/http.HandlerFunc can be devirtualized to a function with
+		// the same underlying type.
+		if !types.Identical(typ, ctyp) {
+			if base.Debug.PGODebug >= 2 {
+				fmt.Printf("%v: edge %s:%d -> %s (weight %d): %v doesn't match %v\n", ir.Line(call), callerName, callOffset, e.Dst.Name(), e.Weight, ctyp, typ)
+			}
+			return false
+		}
+
+		return true
+	})
+}

src/cmd/compile/internal/devirtualize/pgo_test.go

@@ -8,8 +8,8 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/pgo"
-	"cmd/compile/internal/types"
 	"cmd/compile/internal/typecheck"
+	"cmd/compile/internal/types"
 	"cmd/internal/obj"
 	"cmd/internal/src"
 	"testing"
@@ -31,67 +31,82 @@ func makePos(b *src.PosBase, line, col uint) src.XPos {
 	return base.Ctxt.PosTable.XPos(src.MakePos(b, line, col))
 }
 
-func TestFindHotConcreteCallee(t *testing.T) {
+type profileBuilder struct {
+	p *pgo.Profile
+}
+
+func newProfileBuilder() *profileBuilder {
 	// findHotConcreteCallee only uses pgo.Profile.WeightedCG, so we're
 	// going to take a shortcut and only construct that.
-	p := &pgo.Profile{
-		WeightedCG: &pgo.IRGraph{
-			IRNodes: make(map[string]*pgo.IRNode),
+	return &profileBuilder{
+		p: &pgo.Profile{
+			WeightedCG: &pgo.IRGraph{
+				IRNodes: make(map[string]*pgo.IRNode),
+			},
 		},
 	}
+}
 
-	// Create a new IRNode and add it to p.
-	//
-	// fn may be nil, in which case the node will set LinkerSymbolName.
-	newNode := func(name string, fn *ir.Func) *pgo.IRNode {
-		n := &pgo.IRNode{
-			OutEdges: make(map[pgo.NamedCallEdge]*pgo.IREdge),
-		}
-		if fn != nil {
-			n.AST = fn
-		} else {
-			n.LinkerSymbolName = name
-		}
-		p.WeightedCG.IRNodes[name] = n
-		return n
-	}
+// Profile returns the constructed profile.
+func (p *profileBuilder) Profile() *pgo.Profile {
+	return p.p
+}
 
-	// Add a new call edge from caller to callee.
-	addEdge := func(caller, callee *pgo.IRNode, offset int, weight int64) {
-		namedEdge := pgo.NamedCallEdge{
-			CallerName:     caller.Name(),
-			CalleeName:     callee.Name(),
-			CallSiteOffset: offset,
-		}
-		irEdge := &pgo.IREdge{
-			Src:            caller,
-			Dst:            callee,
-			CallSiteOffset: offset,
-			Weight:         weight,
-		}
-		caller.OutEdges[namedEdge] = irEdge
+// NewNode creates a new IRNode and adds it to the profile.
+//
+// fn may be nil, in which case the node will set LinkerSymbolName.
+func (p *profileBuilder) NewNode(name string, fn *ir.Func) *pgo.IRNode {
+	n := &pgo.IRNode{
+		OutEdges: make(map[pgo.NamedCallEdge]*pgo.IREdge),
 	}
+	if fn != nil {
+		n.AST = fn
+	} else {
+		n.LinkerSymbolName = name
+	}
+	p.p.WeightedCG.IRNodes[name] = n
+	return n
+}
+
+// Add a new call edge from caller to callee.
+func addEdge(caller, callee *pgo.IRNode, offset int, weight int64) {
+	namedEdge := pgo.NamedCallEdge{
+		CallerName:     caller.Name(),
+		CalleeName:     callee.Name(),
+		CallSiteOffset: offset,
+	}
+	irEdge := &pgo.IREdge{
+		Src:            caller,
+		Dst:            callee,
+		CallSiteOffset: offset,
+		Weight:         weight,
+	}
+	caller.OutEdges[namedEdge] = irEdge
+}
+
+// Create a new struct type named structName with a method named methName and
+// return the method.
+func makeStructWithMethod(pkg *types.Pkg, structName, methName string) *ir.Func {
+	// type structName struct{}
+	structType := types.NewStruct(nil)
+
+	// func (structName) methodName()
+	recv := types.NewField(src.NoXPos, typecheck.Lookup(structName), structType)
+	sig := types.NewSignature(recv, nil, nil)
+	fn := ir.NewFunc(src.NoXPos, src.NoXPos, pkg.Lookup(structName+"."+methName), sig)
+
+	// Add the method to the struct.
+	structType.SetMethods([]*types.Field{types.NewField(src.NoXPos, typecheck.Lookup(methName), sig)})
+
+	return fn
+}
+
+func TestFindHotConcreteInterfaceCallee(t *testing.T) {
+	p := newProfileBuilder()
 
 	pkgFoo := types.NewPkg("example.com/foo", "foo")
 	basePos := src.NewFileBase("foo.go", "/foo.go")
 
-	// Create a new struct type named structName with a method named methName and
-	// return the method.
-	makeStructWithMethod := func(structName, methName string) *ir.Func {
-		// type structName struct{}
-		structType := types.NewStruct(nil)
-
-		// func (structName) methodName()
-		recv := types.NewField(src.NoXPos, typecheck.Lookup(structName), structType)
-		sig := types.NewSignature(recv, nil, nil)
-		fn := ir.NewFunc(src.NoXPos, src.NoXPos, pkgFoo.Lookup(structName + "." + methName), sig)
-
-		// Add the method to the struct.
-		structType.SetMethods([]*types.Field{types.NewField(src.NoXPos, typecheck.Lookup(methName), sig)})
-
-		return fn
-	}
-
 	const (
 		// Caller start line.
 		callerStart = 42
@@ -112,21 +127,21 @@ func TestFindHotConcreteCallee(t *testing.T) {
 
 	callerFn := ir.NewFunc(makePos(basePos, callerStart, 1), src.NoXPos, pkgFoo.Lookup("Caller"), types.NewSignature(nil, nil, nil))
 
-	hotCalleeFn := makeStructWithMethod("HotCallee", "Foo")
-	coldCalleeFn := makeStructWithMethod("ColdCallee", "Foo")
-	wrongLineCalleeFn := makeStructWithMethod("WrongLineCallee", "Foo")
-	wrongMethodCalleeFn := makeStructWithMethod("WrongMethodCallee", "Bar")
+	hotCalleeFn := makeStructWithMethod(pkgFoo, "HotCallee", "Foo")
+	coldCalleeFn := makeStructWithMethod(pkgFoo, "ColdCallee", "Foo")
+	wrongLineCalleeFn := makeStructWithMethod(pkgFoo, "WrongLineCallee", "Foo")
+	wrongMethodCalleeFn := makeStructWithMethod(pkgFoo, "WrongMethodCallee", "Bar")
 
-	callerNode := newNode("example.com/foo.Caller", callerFn)
-	hotCalleeNode := newNode("example.com/foo.HotCallee.Foo", hotCalleeFn)
-	coldCalleeNode := newNode("example.com/foo.ColdCallee.Foo", coldCalleeFn)
-	wrongLineCalleeNode := newNode("example.com/foo.WrongCalleeLine.Foo", wrongLineCalleeFn)
-	wrongMethodCalleeNode := newNode("example.com/foo.WrongCalleeMethod.Foo", wrongMethodCalleeFn)
+	callerNode := p.NewNode("example.com/foo.Caller", callerFn)
+	hotCalleeNode := p.NewNode("example.com/foo.HotCallee.Foo", hotCalleeFn)
+	coldCalleeNode := p.NewNode("example.com/foo.ColdCallee.Foo", coldCalleeFn)
+	wrongLineCalleeNode := p.NewNode("example.com/foo.WrongCalleeLine.Foo", wrongLineCalleeFn)
+	wrongMethodCalleeNode := p.NewNode("example.com/foo.WrongCalleeMethod.Foo", wrongMethodCalleeFn)
 
-	hotMissingCalleeNode := newNode("example.com/bar.HotMissingCallee.Foo", nil)
+	hotMissingCalleeNode := p.NewNode("example.com/bar.HotMissingCallee.Foo", nil)
 
 	addEdge(callerNode, wrongLineCalleeNode, wrongCallOffset, 100) // Really hot, but wrong line.
-	addEdge(callerNode, wrongMethodCalleeNode, callOffset, 100) // Really hot, but wrong method type.
+	addEdge(callerNode, wrongMethodCalleeNode, callOffset, 100)    // Really hot, but wrong method type.
 	addEdge(callerNode, hotCalleeNode, callOffset, 10)
 	addEdge(callerNode, coldCalleeNode, callOffset, 1)
 
@@ -141,7 +156,7 @@ func TestFindHotConcreteCallee(t *testing.T) {
 	sel := typecheck.NewMethodExpr(src.NoXPos, iface, typecheck.Lookup("Foo"))
 	call := ir.NewCallExpr(makePos(basePos, callerStart+callOffset, 1), ir.OCALLINTER, sel, nil)
 
-	gotFn, gotWeight := findHotConcreteCallee(p, callerFn, call)
+	gotFn, gotWeight := findHotConcreteInterfaceCallee(p.Profile(), callerFn, call)
 	if gotFn != hotCalleeFn {
 		t.Errorf("findHotConcreteInterfaceCallee func got %v want %v", gotFn, hotCalleeFn)
 	}
@@ -149,3 +164,54 @@ func TestFindHotConcreteCallee(t *testing.T) {
 		t.Errorf("findHotConcreteInterfaceCallee weight got %v want 10", gotWeight)
 	}
 }
+
+func TestFindHotConcreteFunctionCallee(t *testing.T) {
+	// TestFindHotConcreteInterfaceCallee already covered basic weight
+	// comparisons, which is shared logic. Here we just test type signature
+	// disambiguation.
+
+	p := newProfileBuilder()
+
+	pkgFoo := types.NewPkg("example.com/foo", "foo")
+	basePos := src.NewFileBase("foo.go", "/foo.go")
+
+	const (
+		// Caller start line.
+		callerStart = 42
+
+		// The line offset of the call we care about.
+		callOffset = 1
+	)
+
+	callerFn := ir.NewFunc(makePos(basePos, callerStart, 1), src.NoXPos, pkgFoo.Lookup("Caller"), types.NewSignature(nil, nil, nil))
+
+	// func HotCallee()
+	hotCalleeFn := ir.NewFunc(src.NoXPos, src.NoXPos, pkgFoo.Lookup("HotCallee"), types.NewSignature(nil, nil, nil))
+
+	// func WrongCallee() bool
+	wrongCalleeFn := ir.NewFunc(src.NoXPos, src.NoXPos, pkgFoo.Lookup("WrongCallee"), types.NewSignature(nil, nil,
+		[]*types.Field{
+			types.NewField(src.NoXPos, nil, types.Types[types.TBOOL]),
+		},
+	))
+
+	callerNode := p.NewNode("example.com/foo.Caller", callerFn)
+	hotCalleeNode := p.NewNode("example.com/foo.HotCallee", hotCalleeFn)
+	wrongCalleeNode := p.NewNode("example.com/foo.WrongCallee", wrongCalleeFn)
+
+	addEdge(callerNode, wrongCalleeNode, callOffset, 100) // Really hot, but wrong function type.
+	addEdge(callerNode, hotCalleeNode, callOffset, 10)
+
+	// var fn func()
+	name := ir.NewNameAt(src.NoXPos, typecheck.Lookup("fn"), types.NewSignature(nil, nil, nil))
+	// fn()
+	call := ir.NewCallExpr(makePos(basePos, callerStart+callOffset, 1), ir.OCALL, name, nil)
+
+	gotFn, gotWeight := findHotConcreteFunctionCallee(p.Profile(), callerFn, call)
+	if gotFn != hotCalleeFn {
+		t.Errorf("findHotConcreteFunctionCallee func got %v want %v", gotFn, hotCalleeFn)
+	}
+	if gotWeight != 10 {
+		t.Errorf("findHotConcreteFunctionCallee weight got %v want 10", gotWeight)
+	}
+}

src/cmd/compile/internal/escape/graph.go

@@ -38,7 +38,7 @@ import (
 //        e.value(k, n.Left)
 //    }
 
-// An location represents an abstract location that stores a Go
+// A location represents an abstract location that stores a Go
 // variable.
 type location struct {
 	n         ir.Node  // represented variable or expression, if any

src/cmd/compile/internal/gc/main.go

@@ -9,10 +9,10 @@ import (
 	"bytes"
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/coverage"
-	"cmd/compile/internal/devirtualize"
 	"cmd/compile/internal/dwarfgen"
 	"cmd/compile/internal/escape"
 	"cmd/compile/internal/inline"
+	"cmd/compile/internal/inline/interleaved"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/logopt"
 	"cmd/compile/internal/loopvar"
@@ -224,30 +224,15 @@ func Main(archInit func(*ssagen.ArchInfo)) {
 		}
 	}
 
-	base.Timer.Start("fe", "pgo-devirtualization")
-	if profile != nil && base.Debug.PGODevirtualize > 0 {
-		// TODO(prattmic): No need to use bottom-up visit order. This
-		// is mirroring the PGO IRGraph visit order, which also need
-		// not be bottom-up.
-		ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) {
-			for _, fn := range list {
-				devirtualize.ProfileGuided(fn, profile)
-			}
-		})
-		ir.CurFunc = nil
-	}
+	// Interleaved devirtualization and inlining.
+	base.Timer.Start("fe", "devirtualize-and-inline")
+	interleaved.DevirtualizeAndInlinePackage(typecheck.Target, profile)
 
-	// Inlining
-	base.Timer.Start("fe", "inlining")
-	if base.Flag.LowerL != 0 {
-		inline.InlinePackage(profile)
-	}
 	noder.MakeWrappers(typecheck.Target) // must happen after inlining
 
-	// Devirtualize and get variable capture right in for loops
+	// Get variable capture right in for loops.
 	var transformed []loopvar.VarAndLoop
 	for _, fn := range typecheck.Target.Funcs {
-		devirtualize.Static(fn)
 		transformed = append(transformed, loopvar.ForCapture(fn)...)
 	}
 	ir.CurFunc = nil

src/cmd/compile/internal/gc/util.go

@@ -5,17 +5,34 @@
 package gc
 
 import (
+	"net/url"
 	"os"
+	"path/filepath"
 	"runtime"
 	"runtime/pprof"
 	tracepkg "runtime/trace"
+	"strings"
 
 	"cmd/compile/internal/base"
 )
 
+func profileName(fn, suffix string) string {
+	if strings.HasSuffix(fn, string(os.PathSeparator)) {
+		err := os.MkdirAll(fn, 0755)
+		if err != nil {
+			base.Fatalf("%v", err)
+		}
+	}
+	if fi, statErr := os.Stat(fn); statErr == nil && fi.IsDir() {
+		fn = filepath.Join(fn, url.PathEscape(base.Ctxt.Pkgpath)+suffix)
+	}
+	return fn
+}
+
 func startProfile() {
 	if base.Flag.CPUProfile != "" {
-		f, err := os.Create(base.Flag.CPUProfile)
+		fn := profileName(base.Flag.CPUProfile, ".cpuprof")
+		f, err := os.Create(fn)
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
@@ -28,18 +45,36 @@ func startProfile() {
 		if base.Flag.MemProfileRate != 0 {
 			runtime.MemProfileRate = base.Flag.MemProfileRate
 		}
-		f, err := os.Create(base.Flag.MemProfile)
+		const (
+			gzipFormat = 0
+			textFormat = 1
+		)
+		// compilebench parses the memory profile to extract memstats,
+		// which are only written in the legacy (text) pprof format.
+		// See golang.org/issue/18641 and runtime/pprof/pprof.go:writeHeap.
+		// gzipFormat is what most people want, otherwise
+		var format = textFormat
+		fn := base.Flag.MemProfile
+		if strings.HasSuffix(fn, string(os.PathSeparator)) {
+			err := os.MkdirAll(fn, 0755)
+			if err != nil {
+				base.Fatalf("%v", err)
+			}
+		}
+		if fi, statErr := os.Stat(fn); statErr == nil && fi.IsDir() {
+			fn = filepath.Join(fn, url.PathEscape(base.Ctxt.Pkgpath)+".memprof")
+			format = gzipFormat
+		}
+
+		f, err := os.Create(fn)
+
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
 		base.AtExit(func() {
 			// Profile all outstanding allocations.
 			runtime.GC()
-			// compilebench parses the memory profile to extract memstats,
-			// which are only written in the legacy pprof format.
-			// See golang.org/issue/18641 and runtime/pprof/pprof.go:writeHeap.
-			const writeLegacyFormat = 1
-			if err := pprof.Lookup("heap").WriteTo(f, writeLegacyFormat); err != nil {
+			if err := pprof.Lookup("heap").WriteTo(f, format); err != nil {
 				base.Fatalf("%v", err)
 			}
 		})
@@ -48,7 +83,7 @@ func startProfile() {
 		runtime.MemProfileRate = 0
 	}
 	if base.Flag.BlockProfile != "" {
-		f, err := os.Create(base.Flag.BlockProfile)
+		f, err := os.Create(profileName(base.Flag.BlockProfile, ".blockprof"))
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
@@ -59,7 +94,7 @@ func startProfile() {
 		})
 	}
 	if base.Flag.MutexProfile != "" {
-		f, err := os.Create(base.Flag.MutexProfile)
+		f, err := os.Create(profileName(base.Flag.MutexProfile, ".mutexprof"))
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
@@ -70,7 +105,7 @@ func startProfile() {
 		})
 	}
 	if base.Flag.TraceProfile != "" {
-		f, err := os.Create(base.Flag.TraceProfile)
+		f, err := os.Create(profileName(base.Flag.TraceProfile, ".trace"))
 		if err != nil {
 			base.Fatalf("%v", err)
 		}

src/cmd/compile/internal/inline/inl.go

@@ -29,7 +29,7 @@ package inline
 import (
 	"fmt"
 	"go/constant"
-	"internal/goexperiment"
+	"internal/buildcfg"
 	"strconv"
 
 	"cmd/compile/internal/base"
@@ -77,8 +77,8 @@ var (
 	inlineHotMaxBudget int32 = 2000
 )
 
-// pgoInlinePrologue records the hot callsites from ir-graph.
-func pgoInlinePrologue(p *pgo.Profile, funcs []*ir.Func) {
+// PGOInlinePrologue records the hot callsites from ir-graph.
+func PGOInlinePrologue(p *pgo.Profile, funcs []*ir.Func) {
 	if base.Debug.PGOInlineCDFThreshold != "" {
 		if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
 			inlineCDFHotCallSiteThresholdPercent = s
@@ -135,73 +135,52 @@ func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NamedCallEdge) {
 	return 0, p.NamedEdgeMap.ByWeight
 }
 
-// InlinePackage finds functions that can be inlined and clones them before walk expands them.
-func InlinePackage(p *pgo.Profile) {
-	if base.Debug.PGOInline == 0 {
-		p = nil
+// CanInlineFuncs computes whether a batch of functions are inlinable.
+func CanInlineFuncs(funcs []*ir.Func, profile *pgo.Profile) {
+	if profile != nil {
+		PGOInlinePrologue(profile, funcs)
 	}
 
-	InlineDecls(p, typecheck.Target.Funcs, true)
-
-	// Perform a garbage collection of hidden closures functions that
-	// are no longer reachable from top-level functions following
-	// inlining. See #59404 and #59638 for more context.
-	garbageCollectUnreferencedHiddenClosures()
-
-	if base.Debug.DumpInlFuncProps != "" {
-		inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps, nil)
-	}
-	if goexperiment.NewInliner {
-		postProcessCallSites(p)
-	}
+	ir.VisitFuncsBottomUp(funcs, func(list []*ir.Func, recursive bool) {
+		CanInlineSCC(list, recursive, profile)
+	})
 }
 
-// InlineDecls applies inlining to the given batch of declarations.
-func InlineDecls(p *pgo.Profile, funcs []*ir.Func, doInline bool) {
-	if p != nil {
-		pgoInlinePrologue(p, funcs)
+// CanInlineSCC computes the inlinability of functions within an SCC
+// (strongly connected component).
+//
+// CanInlineSCC is designed to be used by ir.VisitFuncsBottomUp
+// callbacks.
+func CanInlineSCC(funcs []*ir.Func, recursive bool, profile *pgo.Profile) {
+	if base.Flag.LowerL == 0 {
+		return
 	}
 
-	doCanInline := func(n *ir.Func, recursive bool, numfns int) {
+	numfns := numNonClosures(funcs)
+
+	for _, fn := range funcs {
 		if !recursive || numfns > 1 {
 			// We allow inlining if there is no
 			// recursion, or the recursion cycle is
 			// across more than one function.
-			CanInline(n, p)
+			CanInline(fn, profile)
 		} else {
-			if base.Flag.LowerM > 1 && n.OClosure == nil {
-				fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname)
+			if base.Flag.LowerM > 1 && fn.OClosure == nil {
+				fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(fn), fn.Nname)
 			}
 		}
+		if inlheur.Enabled() {
+			analyzeFuncProps(fn, profile)
+		}
 	}
-
-	ir.VisitFuncsBottomUp(funcs, func(list []*ir.Func, recursive bool) {
-		numfns := numNonClosures(list)
-		// We visit functions within an SCC in fairly arbitrary order,
-		// so by computing inlinability for all functions in the SCC
-		// before performing any inlining, the results are less
-		// sensitive to the order within the SCC (see #58905 for an
-		// example).
-
-		// First compute inlinability for all functions in the SCC ...
-		for _, n := range list {
-			doCanInline(n, recursive, numfns)
-		}
-		// ... then make a second pass to do inlining of calls.
-		if doInline {
-			for _, n := range list {
-				InlineCalls(n, p)
-			}
-		}
-	})
 }
 
-// garbageCollectUnreferencedHiddenClosures makes a pass over all the
+// GarbageCollectUnreferencedHiddenClosures makes a pass over all the
 // top-level (non-hidden-closure) functions looking for nested closure
 // functions that are reachable, then sweeps through the Target.Decls
 // list and marks any non-reachable hidden closure function as dead.
 // See issues #59404 and #59638 for more context.
-func garbageCollectUnreferencedHiddenClosures() {
+func GarbageCollectUnreferencedHiddenClosures() {
 
 	liveFuncs := make(map[*ir.Func]bool)
 
@@ -268,7 +247,7 @@ func inlineBudget(fn *ir.Func, profile *pgo.Profile, relaxed bool, verbose bool)
 		}
 	}
 	if relaxed {
-		budget += inlineMaxBudget
+		budget += inlheur.BudgetExpansion(inlineMaxBudget)
 	}
 	return budget
 }
@@ -281,12 +260,6 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
 		base.Fatalf("CanInline no nname %+v", fn)
 	}
 
-	var funcProps *inlheur.FuncProps
-	if goexperiment.NewInliner || inlheur.UnitTesting() {
-		funcProps = inlheur.AnalyzeFunc(fn,
-			func(fn *ir.Func) { CanInline(fn, profile) })
-	}
-
 	var reason string // reason, if any, that the function was not inlined
 	if base.Flag.LowerM > 1 || logopt.Enabled() {
 		defer func() {
@@ -320,11 +293,8 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
 		cc = 1 // this appears to yield better performance than 0.
 	}
 
-	// Used a "relaxed" inline budget if goexperiment.NewInliner is in
-	// effect, or if we're producing a debugging dump.
-	relaxed := goexperiment.NewInliner ||
-		(base.Debug.DumpInlFuncProps != "" ||
-			base.Debug.DumpInlCallSiteScores != 0)
+	// Used a "relaxed" inline budget if the new inliner is enabled.
+	relaxed := inlheur.Enabled()
 
 	// Compute the inline budget for this func.
 	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
@@ -340,7 +310,7 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
 
 	visitor := hairyVisitor{
 		curFunc:       fn,
-		isBigFunc:     isBigFunc(fn),
+		isBigFunc:     IsBigFunc(fn),
 		budget:        budget,
 		maxBudget:     budget,
 		extraCallCost: cc,
@@ -358,17 +328,22 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
 
 		CanDelayResults: canDelayResults(fn),
 	}
-	if goexperiment.NewInliner {
-		n.Func.Inl.Properties = funcProps.SerializeToString()
+	if base.Flag.LowerM != 0 || logopt.Enabled() {
+		noteInlinableFunc(n, fn, budget-visitor.budget)
 	}
+}
 
+// noteInlinableFunc issues a message to the user that the specified
+// function is inlinable.
+func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
 	if base.Flag.LowerM > 1 {
-		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(fn.Body))
+		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, cost, fn.Type(), ir.Nodes(fn.Body))
 	} else if base.Flag.LowerM != 0 {
 		fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
 	}
+	// JSON optimization log output.
 	if logopt.Enabled() {
-		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", budget-visitor.budget))
+		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))
 	}
 }
 
@@ -527,6 +502,8 @@ opSwitch:
 				case "throw":
 					v.budget -= inlineExtraThrowCost
 					break opSwitch
+				case "panicrangeexit":
+					cheap = true
 				}
 				// Special case for reflect.noescape. It does just type
 				// conversions to appease the escape analysis, and doesn't
@@ -590,7 +567,7 @@ opSwitch:
 			// Check whether we'd actually inline this call. Set
 			// log == false since we aren't actually doing inlining
 			// yet.
-			if canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false) {
+			if ok, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok {
 				// mkinlcall would inline this call [1], so use
 				// the cost of the inline body as the cost of
 				// the call, as that is what will actually
@@ -737,14 +714,16 @@ opSwitch:
 		// particular, to avoid breaking the existing inlinability regress
 		// tests), we need to compensate for this here.
 		//
-		// See also identical logic in isBigFunc.
-		if init := n.Rhs[0].Init(); len(init) == 1 {
-			if _, ok := init[0].(*ir.AssignListStmt); ok {
-				// 4 for each value, because each temporary variable now
-				// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.
-				//
-				// 1 for the extra "tmp1, tmp2 = f()" assignment statement.
-				v.budget += 4*int32(len(n.Lhs)) + 1
+		// See also identical logic in IsBigFunc.
+		if len(n.Rhs) > 0 {
+			if init := n.Rhs[0].Init(); len(init) == 1 {
+				if _, ok := init[0].(*ir.AssignListStmt); ok {
+					// 4 for each value, because each temporary variable now
+					// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.
+					//
+					// 1 for the extra "tmp1, tmp2 = f()" assignment statement.
+					v.budget += 4*int32(len(n.Lhs)) + 1
+				}
 			}
 		}
 
@@ -776,12 +755,15 @@ opSwitch:
 	return ir.DoChildren(n, v.do)
 }
 
-func isBigFunc(fn *ir.Func) bool {
+// IsBigFunc reports whether fn is a "big" function.
+//
+// Note: The criteria for "big" is heuristic and subject to change.
+func IsBigFunc(fn *ir.Func) bool {
 	budget := inlineBigFunctionNodes
 	return ir.Any(fn, func(n ir.Node) bool {
 		// See logic in hairyVisitor.doNode, explaining unified IR's
 		// handling of "a, b = f()" assignments.
-		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 {
+		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {
 			if init := n.Rhs[0].Init(); len(init) == 1 {
 				if _, ok := init[0].(*ir.AssignListStmt); ok {
 					budget += 4*len(n.Lhs) + 1
@@ -794,128 +776,40 @@ func isBigFunc(fn *ir.Func) bool {
 	})
 }
 
-// InlineCalls/inlnode walks fn's statements and expressions and substitutes any
-// calls made to inlineable functions. This is the external entry point.
-func InlineCalls(fn *ir.Func, profile *pgo.Profile) {
-	if goexperiment.NewInliner && !fn.Wrapper() {
-		inlheur.ScoreCalls(fn)
+// TryInlineCall returns an inlined call expression for call, or nil
+// if inlining is not possible.
+func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgo.Profile) *ir.InlinedCallExpr {
+	if base.Flag.LowerL == 0 {
+		return nil
 	}
-	if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
-		inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps,
-			func(fn *ir.Func) { CanInline(fn, profile) })
+	if call.Op() != ir.OCALLFUNC {
+		return nil
 	}
-	savefn := ir.CurFunc
-	ir.CurFunc = fn
-	bigCaller := isBigFunc(fn)
-	if bigCaller && base.Flag.LowerM > 1 {
-		fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
-	}
-	var inlCalls []*ir.InlinedCallExpr
-	var edit func(ir.Node) ir.Node
-	edit = func(n ir.Node) ir.Node {
-		return inlnode(fn, n, bigCaller, &inlCalls, edit, profile)
-	}
-	ir.EditChildren(fn, edit)
-
-	// If we inlined any calls, we want to recursively visit their
-	// bodies for further inlining. However, we need to wait until
-	// *after* the original function body has been expanded, or else
-	// inlCallee can have false positives (e.g., #54632).
-	for len(inlCalls) > 0 {
-		call := inlCalls[0]
-		inlCalls = inlCalls[1:]
-		ir.EditChildren(call, edit)
+	if call.GoDefer || call.NoInline {
+		return nil
 	}
 
-	ir.CurFunc = savefn
-}
-
-// inlnode recurses over the tree to find inlineable calls, which will
-// be turned into OINLCALLs by mkinlcall. When the recursion comes
-// back up will examine left, right, list, rlist, ninit, ntest, nincr,
-// nbody and nelse and use one of the 4 inlconv/glue functions above
-// to turn the OINLCALL into an expression, a statement, or patch it
-// in to this nodes list or rlist as appropriate.
-// NOTE it makes no sense to pass the glue functions down the
-// recursion to the level where the OINLCALL gets created because they
-// have to edit /this/ n, so you'd have to push that one down as well,
-// but then you may as well do it here.  so this is cleaner and
-// shorter and less complicated.
-// The result of inlnode MUST be assigned back to n, e.g.
-//
-//	n.Left = inlnode(n.Left)
-func inlnode(callerfn *ir.Func, n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node {
-	if n == nil {
-		return n
-	}
-
-	switch n.Op() {
-	case ir.ODEFER, ir.OGO:
-		n := n.(*ir.GoDeferStmt)
-		switch call := n.Call; call.Op() {
-		case ir.OCALLMETH:
-			base.FatalfAt(call.Pos(), "OCALLMETH missed by typecheck")
-		case ir.OCALLFUNC:
-			call := call.(*ir.CallExpr)
-			call.NoInline = true
-		}
-	case ir.OTAILCALL:
-		n := n.(*ir.TailCallStmt)
-		n.Call.NoInline = true // Not inline a tail call for now. Maybe we could inline it just like RETURN fn(arg)?
-
-	// TODO do them here (or earlier),
-	// so escape analysis can avoid more heapmoves.
-	case ir.OCLOSURE:
-		return n
-	case ir.OCALLMETH:
-		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
-	case ir.OCALLFUNC:
-		n := n.(*ir.CallExpr)
-		if n.Fun.Op() == ir.OMETHEXPR {
-			// Prevent inlining some reflect.Value methods when using checkptr,
-			// even when package reflect was compiled without it (#35073).
-			if meth := ir.MethodExprName(n.Fun); meth != nil {
-				s := meth.Sym()
-				if base.Debug.Checkptr != 0 {
-					switch types.ReflectSymName(s) {
-					case "Value.UnsafeAddr", "Value.Pointer":
-						return n
-					}
-				}
+	// Prevent inlining some reflect.Value methods when using checkptr,
+	// even when package reflect was compiled without it (#35073).
+	if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {
+		if method := ir.MethodExprName(call.Fun); method != nil {
+			switch types.ReflectSymName(method.Sym()) {
+			case "Value.UnsafeAddr", "Value.Pointer":
+				return nil
 			}
 		}
 	}
 
-	lno := ir.SetPos(n)
-
-	ir.EditChildren(n, edit)
-
-	// with all the branches out of the way, it is now time to
-	// transmogrify this node itself unless inhibited by the
-	// switch at the top of this function.
-	switch n.Op() {
-	case ir.OCALLMETH:
-		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
-
-	case ir.OCALLFUNC:
-		call := n.(*ir.CallExpr)
-		if call.NoInline {
-			break
-		}
-		if base.Flag.LowerM > 3 {
-			fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.Fun)
-		}
-		if ir.IsIntrinsicCall(call) {
-			break
-		}
-		if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
-			n = mkinlcall(callerfn, call, fn, bigCaller, inlCalls)
-		}
+	if base.Flag.LowerM > 3 {
+		fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
 	}
-
-	base.Pos = lno
-
-	return n
+	if ir.IsIntrinsicCall(call) {
+		return nil
+	}
+	if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
+		return mkinlcall(callerfn, call, fn, bigCaller)
+	}
+	return nil
 }
 
 // inlCallee takes a function-typed expression and returns the underlying function ONAME
@@ -966,9 +860,10 @@ var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInde
 // inlineCostOK returns true if call n from caller to callee is cheap enough to
 // inline. bigCaller indicates that caller is a big function.
 //
-// If inlineCostOK returns false, it also returns the max cost that the callee
-// exceeded.
-func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32) {
+// In addition to the "cost OK" boolean, it also returns the "max
+// cost" limit used to make the decision (which may differ depending
+// on func size), and the score assigned to this specific callsite.
+func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32) {
 	maxCost := int32(inlineMaxBudget)
 	if bigCaller {
 		// We use this to restrict inlining into very big functions.
@@ -977,17 +872,16 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool
 	}
 
 	metric := callee.Inl.Cost
-	if goexperiment.NewInliner {
-		ok, score := inlheur.GetCallSiteScore(n)
+	if inlheur.Enabled() {
+		score, ok := inlheur.GetCallSiteScore(caller, n)
 		if ok {
 			metric = int32(score)
 		}
-
 	}
 
 	if metric <= maxCost {
 		// Simple case. Function is already cheap enough.
-		return true, 0
+		return true, 0, metric
 	}
 
 	// We'll also allow inlining of hot functions below inlineHotMaxBudget,
@@ -997,7 +891,7 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool
 	csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
 	if _, ok := candHotEdgeMap[csi]; !ok {
 		// Cold
-		return false, maxCost
+		return false, maxCost, metric
 	}
 
 	// Hot
@@ -1006,47 +900,49 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool
 		if base.Debug.PGODebug > 0 {
 			fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
 		}
-		return false, maxCost
+		return false, maxCost, metric
 	}
 
 	if metric > inlineHotMaxBudget {
-		return false, inlineHotMaxBudget
+		return false, inlineHotMaxBudget, metric
 	}
 
 	if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
 		// De-selected by PGO Hash.
-		return false, maxCost
+		return false, maxCost, metric
 	}
 
 	if base.Debug.PGODebug > 0 {
 		fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
 	}
 
-	return true, 0
+	return true, 0, metric
 }
 
-// canInlineCallsite returns true if the call n from caller to callee can be
-// inlined. bigCaller indicates that caller is a big function. log indicates
-// that the 'cannot inline' reason should be logged.
+// canInlineCallsite returns true if the call n from caller to callee
+// can be inlined, plus the score computed for the call expr in
+// question. bigCaller indicates that caller is a big function. log
+// indicates that the 'cannot inline' reason should be logged.
 //
 // Preconditions: CanInline(callee) has already been called.
-func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) bool {
+func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32) {
 	if callee.Inl == nil {
 		// callee is never inlinable.
 		if log && logopt.Enabled() {
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
 				fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
 		}
-		return false
+		return false, 0
 	}
 
-	if ok, maxCost := inlineCostOK(n, callerfn, callee, bigCaller); !ok {
+	ok, maxCost, callSiteScore := inlineCostOK(n, callerfn, callee, bigCaller)
+	if !ok {
 		// callee cost too high for this call site.
 		if log && logopt.Enabled() {
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
 				fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
 		}
-		return false
+		return false, 0
 	}
 
 	if callee == callerfn {
@@ -1054,7 +950,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 		if log && logopt.Enabled() {
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
 		}
-		return false
+		return false, 0
 	}
 
 	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
@@ -1068,7 +964,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
 				fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
 		}
-		return false
+		return false, 0
 	}
 
 	if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
@@ -1076,7 +972,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
 				fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
 		}
-		return false
+		return false, 0
 	}
 
 	// Check if we've already inlined this function at this particular
@@ -1099,24 +995,24 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 						fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
 				}
 			}
-			return false
+			return false, 0
 		}
 	}
 
-	return true
+	return true, callSiteScore
 }
 
-// If n is a OCALLFUNC node, and fn is an ONAME node for a
-// function with an inlinable body, return an OINLCALL node that can replace n.
-// The returned node's Ninit has the parameter assignments, the Nbody is the
-// inlined function body, and (List, Rlist) contain the (input, output)
-// parameters.
+// mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or
+// nil if it cannot be inlined. callerfn is the function that contains
+// n, and fn is the function being called.
+//
 // The result of mkinlcall MUST be assigned back to n, e.g.
 //
 //	n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
-	if !canInlineCallExpr(callerfn, n, fn, bigCaller, true) {
-		return n
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool) *ir.InlinedCallExpr {
+	ok, score := canInlineCallExpr(callerfn, n, fn, bigCaller, true)
+	if !ok {
+		return nil
 	}
 	typecheck.AssertFixedCall(n)
 
@@ -1174,7 +1070,12 @@ func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, i
 	}
 
 	if base.Flag.LowerM != 0 {
-		fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
+		if buildcfg.Experiment.NewInliner {
+			fmt.Printf("%v: inlining call to %v with score %d\n",
+				ir.Line(n), fn, score)
+		} else {
+			fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
+		}
 	}
 	if base.Flag.LowerM > 2 {
 		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
@@ -1190,7 +1091,9 @@ func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, i
 		fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)
 	}
 
-	*inlCalls = append(*inlCalls, res)
+	if inlheur.Enabled() {
+		inlheur.UpdateCallsiteTable(callerfn, n, res)
+	}
 
 	return res
 }
@@ -1295,7 +1198,7 @@ func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
 	return v
 }
 
-func postProcessCallSites(profile *pgo.Profile) {
+func PostProcessCallSites(profile *pgo.Profile) {
 	if base.Debug.DumpInlCallSiteScores != 0 {
 		budgetCallback := func(fn *ir.Func, prof *pgo.Profile) (int32, bool) {
 			v := inlineBudget(fn, prof, false, false)
@@ -1304,3 +1207,11 @@ func postProcessCallSites(profile *pgo.Profile) {
 		inlheur.DumpInlCallSiteScores(profile, budgetCallback)
 	}
 }
+
+func analyzeFuncProps(fn *ir.Func, p *pgo.Profile) {
+	canInline := func(fn *ir.Func) { CanInline(fn, p) }
+	budgetForFunc := func(fn *ir.Func) int32 {
+		return inlineBudget(fn, p, true, false)
+	}
+	inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)
+}

src/cmd/compile/internal/inline/inlheur/actualexprpropbits_string.go

@@ -0,0 +1,58 @@
+// Code generated by "stringer -bitset -type ActualExprPropBits"; DO NOT EDIT.
+
+package inlheur
+
+import "strconv"
+import "bytes"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ActualExprConstant-1]
+	_ = x[ActualExprIsConcreteConvIface-2]
+	_ = x[ActualExprIsFunc-4]
+	_ = x[ActualExprIsInlinableFunc-8]
+}
+
+var _ActualExprPropBits_value = [...]uint64{
+	0x1, /* ActualExprConstant */
+	0x2, /* ActualExprIsConcreteConvIface */
+	0x4, /* ActualExprIsFunc */
+	0x8, /* ActualExprIsInlinableFunc */
+}
+
+const _ActualExprPropBits_name = "ActualExprConstantActualExprIsConcreteConvIfaceActualExprIsFuncActualExprIsInlinableFunc"
+
+var _ActualExprPropBits_index = [...]uint8{0, 18, 47, 63, 88}
+
+func (i ActualExprPropBits) String() string {
+	var b bytes.Buffer
+
+	remain := uint64(i)
+	seen := false
+
+	for k, v := range _ActualExprPropBits_value {
+		x := _ActualExprPropBits_name[_ActualExprPropBits_index[k]:_ActualExprPropBits_index[k+1]]
+		if v == 0 {
+			if i == 0 {
+				b.WriteString(x)
+				return b.String()
+			}
+			continue
+		}
+		if (v & remain) == v {
+			remain &^= v
+			x := _ActualExprPropBits_name[_ActualExprPropBits_index[k]:_ActualExprPropBits_index[k+1]]
+			if seen {
+				b.WriteString("|")
+			}
+			seen = true
+			b.WriteString(x)
+		}
+	}
+	if remain == 0 {
+		return b.String()
+	}
+	return "ActualExprPropBits(0x" + strconv.FormatInt(int64(i), 16) + ")"
+}

src/cmd/compile/internal/inline/inlheur/analyze.go

@@ -10,7 +10,7 @@ import (
 	"cmd/compile/internal/types"
 	"encoding/json"
 	"fmt"
-	"internal/goexperiment"
+	"internal/buildcfg"
 	"io"
 	"os"
 	"path/filepath"
@@ -40,7 +40,7 @@ const (
 type propAnalyzer interface {
 	nodeVisitPre(n ir.Node)
 	nodeVisitPost(n ir.Node)
-	setResults(fp *FuncProps)
+	setResults(funcProps *FuncProps)
 }
 
 // fnInlHeur contains inline heuristics state information about a
@@ -51,62 +51,135 @@ type propAnalyzer interface {
 // parsing a dump. This is the reason why we have file/fname/line
 // fields below instead of just an *ir.Func field.
 type fnInlHeur struct {
+	props *FuncProps
+	cstab CallSiteTab
 	fname string
 	file  string
 	line  uint
-	props *FuncProps
-	cstab CallSiteTab
 }
 
 var fpmap = map[*ir.Func]fnInlHeur{}
 
-func AnalyzeFunc(fn *ir.Func, canInline func(*ir.Func)) *FuncProps {
-	if fih, ok := fpmap[fn]; ok {
-		return fih.props
+// AnalyzeFunc computes function properties for fn and its contained
+// closures, updating the global 'fpmap' table. It is assumed that
+// "CanInline" has been run on fn and on the closures that feed
+// directly into calls; other closures not directly called will also
+// be checked inlinability for inlinability here in case they are
+// returned as a result.
+func AnalyzeFunc(fn *ir.Func, canInline func(*ir.Func), budgetForFunc func(*ir.Func) int32, inlineMaxBudget int) {
+	if fpmap == nil {
+		// If fpmap is nil this indicates that the main inliner pass is
+		// complete and we're doing inlining of wrappers (no heuristics
+		// used here).
+		return
 	}
-	fp, fcstab := computeFuncProps(fn, canInline)
+	if fn.OClosure != nil {
+		// closures will be processed along with their outer enclosing func.
+		return
+	}
+	enableDebugTraceIfEnv()
+	if debugTrace&debugTraceFuncs != 0 {
+		fmt.Fprintf(os.Stderr, "=-= AnalyzeFunc(%v)\n", fn)
+	}
+	// Build up a list containing 'fn' and any closures it contains. Along
+	// the way, test to see whether each closure is inlinable in case
+	// we might be returning it.
+	funcs := []*ir.Func{fn}
+	ir.VisitFuncAndClosures(fn, func(n ir.Node) {
+		if clo, ok := n.(*ir.ClosureExpr); ok {
+			funcs = append(funcs, clo.Func)
+		}
+	})
+
+	// Analyze the list of functions. We want to visit a given func
+	// only after the closures it contains have been processed, so
+	// iterate through the list in reverse order. Once a function has
+	// been analyzed, revisit the question of whether it should be
+	// inlinable; if it is over the default hairyness limit and it
+	// doesn't have any interesting properties, then we don't want
+	// the overhead of writing out its inline body.
+	nameFinder := newNameFinder(fn)
+	for i := len(funcs) - 1; i >= 0; i-- {
+		f := funcs[i]
+		if f.OClosure != nil && !f.InlinabilityChecked() {
+			canInline(f)
+		}
+		funcProps := analyzeFunc(f, inlineMaxBudget, nameFinder)
+		revisitInlinability(f, funcProps, budgetForFunc)
+		if f.Inl != nil {
+			f.Inl.Properties = funcProps.SerializeToString()
+		}
+	}
+	disableDebugTrace()
+}
+
+// TearDown is invoked at the end of the main inlining pass; doing
+// function analysis and call site scoring is unlikely to help a lot
+// after this point, so nil out fpmap and other globals to reclaim
+// storage.
+func TearDown() {
+	fpmap = nil
+	scoreCallsCache.tab = nil
+	scoreCallsCache.csl = nil
+}
+
+func analyzeFunc(fn *ir.Func, inlineMaxBudget int, nf *nameFinder) *FuncProps {
+	if funcInlHeur, ok := fpmap[fn]; ok {
+		return funcInlHeur.props
+	}
+	funcProps, fcstab := computeFuncProps(fn, inlineMaxBudget, nf)
 	file, line := fnFileLine(fn)
 	entry := fnInlHeur{
 		fname: fn.Sym().Name,
 		file:  file,
 		line:  line,
-		props: fp,
+		props: funcProps,
 		cstab: fcstab,
 	}
-	// Merge this functions call sites into the package level table.
-	if err := cstab.merge(fcstab); err != nil {
-		base.FatalfAt(fn.Pos(), "%v", err)
-	}
 	fn.SetNeverReturns(entry.props.Flags&FuncPropNeverReturns != 0)
 	fpmap[fn] = entry
 	if fn.Inl != nil && fn.Inl.Properties == "" {
 		fn.Inl.Properties = entry.props.SerializeToString()
 	}
-	return fp
+	return funcProps
+}
+
+// revisitInlinability revisits the question of whether to continue to
+// treat function 'fn' as an inline candidate based on the set of
+// properties we've computed for it. If (for example) it has an
+// initial size score of 150 and no interesting properties to speak
+// of, then there isn't really any point to moving ahead with it as an
+// inline candidate.
+func revisitInlinability(fn *ir.Func, funcProps *FuncProps, budgetForFunc func(*ir.Func) int32) {
+	if fn.Inl == nil {
+		return
+	}
+	maxAdj := int32(LargestNegativeScoreAdjustment(fn, funcProps))
+	budget := budgetForFunc(fn)
+	if fn.Inl.Cost+maxAdj > budget {
+		fn.Inl = nil
+	}
 }
 
 // computeFuncProps examines the Go function 'fn' and computes for it
 // a function "properties" object, to be used to drive inlining
 // heuristics. See comments on the FuncProps type for more info.
-func computeFuncProps(fn *ir.Func, canInline func(*ir.Func)) (*FuncProps, CallSiteTab) {
-	enableDebugTraceIfEnv()
+func computeFuncProps(fn *ir.Func, inlineMaxBudget int, nf *nameFinder) (*FuncProps, CallSiteTab) {
 	if debugTrace&debugTraceFuncs != 0 {
 		fmt.Fprintf(os.Stderr, "=-= starting analysis of func %v:\n%+v\n",
-			fn.Sym().Name, fn)
+			fn, fn)
 	}
-	ra := makeResultsAnalyzer(fn, canInline)
-	pa := makeParamsAnalyzer(fn)
+	funcProps := new(FuncProps)
 	ffa := makeFuncFlagsAnalyzer(fn)
-	analyzers := []propAnalyzer{ffa, ra, pa}
-	fp := new(FuncProps)
+	analyzers := []propAnalyzer{ffa}
+	analyzers = addResultsAnalyzer(fn, analyzers, funcProps, inlineMaxBudget, nf)
+	analyzers = addParamsAnalyzer(fn, analyzers, funcProps, nf)
 	runAnalyzersOnFunction(fn, analyzers)
 	for _, a := range analyzers {
-		a.setResults(fp)
+		a.setResults(funcProps)
 	}
-	// Now build up a partial table of callsites for this func.
-	cstab := computeCallSiteTable(fn, ffa.panicPathTable())
-	disableDebugTrace()
-	return fp, cstab
+	cstab := computeCallSiteTable(fn, fn.Body, nil, ffa.panicPathTable(), 0, nf)
+	return funcProps, cstab
 }
 
 func runAnalyzersOnFunction(fn *ir.Func, analyzers []propAnalyzer) {
@@ -125,8 +198,8 @@ func runAnalyzersOnFunction(fn *ir.Func, analyzers []propAnalyzer) {
 }
 
 func propsForFunc(fn *ir.Func) *FuncProps {
-	if fih, ok := fpmap[fn]; ok {
-		return fih.props
+	if funcInlHeur, ok := fpmap[fn]; ok {
+		return funcInlHeur.props
 	} else if fn.Inl != nil && fn.Inl.Properties != "" {
 		// FIXME: considering adding some sort of cache or table
 		// for deserialized properties of imported functions.
@@ -140,32 +213,32 @@ func fnFileLine(fn *ir.Func) (string, uint) {
 	return filepath.Base(p.Filename()), p.Line()
 }
 
+func Enabled() bool {
+	return buildcfg.Experiment.NewInliner || UnitTesting()
+}
+
 func UnitTesting() bool {
-	return base.Debug.DumpInlFuncProps != ""
+	return base.Debug.DumpInlFuncProps != "" ||
+		base.Debug.DumpInlCallSiteScores != 0
 }
 
 // DumpFuncProps computes and caches function properties for the func
-// 'fn' and any closures it contains, or if fn is nil, it writes out the
-// cached set of properties to the file given in 'dumpfile'. Used for
-// the "-d=dumpinlfuncprops=..." command line flag, intended for use
+// 'fn', writing out a description of the previously computed set of
+// properties to the file given in 'dumpfile'. Used for the
+// "-d=dumpinlfuncprops=..." command line flag, intended for use
 // primarily in unit testing.
-func DumpFuncProps(fn *ir.Func, dumpfile string, canInline func(*ir.Func)) {
+func DumpFuncProps(fn *ir.Func, dumpfile string) {
 	if fn != nil {
-		enableDebugTraceIfEnv()
-		dmp := func(fn *ir.Func) {
-			if !goexperiment.NewInliner {
-				ScoreCalls(fn)
-			}
-			captureFuncDumpEntry(fn, canInline)
+		if fn.OClosure != nil {
+			// closures will be processed along with their outer enclosing func.
+			return
 		}
-		captureFuncDumpEntry(fn, canInline)
-		dmp(fn)
-		ir.Visit(fn, func(n ir.Node) {
+		captureFuncDumpEntry(fn)
+		ir.VisitFuncAndClosures(fn, func(n ir.Node) {
 			if clo, ok := n.(*ir.ClosureExpr); ok {
-				dmp(clo.Func)
+				captureFuncDumpEntry(clo.Func)
 			}
 		})
-		disableDebugTrace()
 	} else {
 		emitDumpToFile(dumpfile)
 	}
@@ -221,33 +294,28 @@ func emitDumpToFile(dumpfile string) {
 // and enqueues it for later dumping. Used for the
 // "-d=dumpinlfuncprops=..." command line flag, intended for use
 // primarily in unit testing.
-func captureFuncDumpEntry(fn *ir.Func, canInline func(*ir.Func)) {
+func captureFuncDumpEntry(fn *ir.Func) {
 	// avoid capturing compiler-generated equality funcs.
 	if strings.HasPrefix(fn.Sym().Name, ".eq.") {
 		return
 	}
-	fih, ok := fpmap[fn]
-	// Props object should already be present, unless this is a
-	// directly recursive routine.
+	funcInlHeur, ok := fpmap[fn]
 	if !ok {
-		AnalyzeFunc(fn, canInline)
-		fih = fpmap[fn]
-		if fn.Inl != nil && fn.Inl.Properties == "" {
-			fn.Inl.Properties = fih.props.SerializeToString()
-		}
+		// Missing entry is expected for functions that are too large
+		// to inline. We still want to write out call site scores in
+		// this case however.
+		funcInlHeur = fnInlHeur{cstab: callSiteTab}
 	}
 	if dumpBuffer == nil {
 		dumpBuffer = make(map[*ir.Func]fnInlHeur)
 	}
 	if _, ok := dumpBuffer[fn]; ok {
-		// we can wind up seeing closures multiple times here,
-		// so don't add them more than once.
 		return
 	}
 	if debugTrace&debugTraceFuncs != 0 {
 		fmt.Fprintf(os.Stderr, "=-= capturing dump for %v:\n", fn)
 	}
-	dumpBuffer[fn] = fih
+	dumpBuffer[fn] = funcInlHeur
 }
 
 // dumpFilePreamble writes out a file-level preamble for a given
@@ -263,17 +331,17 @@ func dumpFilePreamble(w io.Writer) {
 // Go function as part of a function properties dump. See the
 // README.txt file in testdata/props for more on the format of
 // this preamble.
-func dumpFnPreamble(w io.Writer, fih *fnInlHeur, ecst encodedCallSiteTab, idx, atl uint) error {
+func dumpFnPreamble(w io.Writer, funcInlHeur *fnInlHeur, ecst encodedCallSiteTab, idx, atl uint) error {
 	fmt.Fprintf(w, "// %s %s %d %d %d\n",
-		fih.file, fih.fname, fih.line, idx, atl)
+		funcInlHeur.file, funcInlHeur.fname, funcInlHeur.line, idx, atl)
 	// emit props as comments, followed by delimiter
-	fmt.Fprintf(w, "%s// %s\n", fih.props.ToString("// "), comDelimiter)
-	data, err := json.Marshal(fih.props)
+	fmt.Fprintf(w, "%s// %s\n", funcInlHeur.props.ToString("// "), comDelimiter)
+	data, err := json.Marshal(funcInlHeur.props)
 	if err != nil {
 		return fmt.Errorf("marshall error %v\n", err)
 	}
 	fmt.Fprintf(w, "// %s\n", string(data))
-	dumpCallSiteComments(w, fih.cstab, ecst)
+	dumpCallSiteComments(w, funcInlHeur.cstab, ecst)
 	fmt.Fprintf(w, "// %s\n", fnDelimiter)
 	return nil
 }

src/cmd/compile/internal/inline/inlheur/analyze_func_callsites.go

@@ -5,52 +5,70 @@
 package inlheur
 
 import (
-	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/pgo"
+	"cmd/compile/internal/typecheck"
 	"fmt"
 	"os"
-	"sort"
 	"strings"
 )
 
 type callSiteAnalyzer struct {
+	fn *ir.Func
+	*nameFinder
+}
+
+type callSiteTableBuilder struct {
+	fn *ir.Func
+	*nameFinder
 	cstab    CallSiteTab
-	fn       *ir.Func
 	ptab     map[ir.Node]pstate
 	nstack   []ir.Node
 	loopNest int
 	isInit   bool
 }
 
-func makeCallSiteAnalyzer(fn *ir.Func, ptab map[ir.Node]pstate) *callSiteAnalyzer {
-	isInit := fn.IsPackageInit() || strings.HasPrefix(fn.Sym().Name, "init.")
+func makeCallSiteAnalyzer(fn *ir.Func) *callSiteAnalyzer {
 	return &callSiteAnalyzer{
-		fn:     fn,
-		cstab:  make(CallSiteTab),
-		ptab:   ptab,
-		isInit: isInit,
+		fn:         fn,
+		nameFinder: newNameFinder(fn),
 	}
 }
 
-func computeCallSiteTable(fn *ir.Func, ptab map[ir.Node]pstate) CallSiteTab {
-	if debugTrace != 0 {
-		fmt.Fprintf(os.Stderr, "=-= making callsite table for func %v:\n",
-			fn.Sym().Name)
+func makeCallSiteTableBuilder(fn *ir.Func, cstab CallSiteTab, ptab map[ir.Node]pstate, loopNestingLevel int, nf *nameFinder) *callSiteTableBuilder {
+	isInit := fn.IsPackageInit() || strings.HasPrefix(fn.Sym().Name, "init.")
+	return &callSiteTableBuilder{
+		fn:         fn,
+		cstab:      cstab,
+		ptab:       ptab,
+		isInit:     isInit,
+		loopNest:   loopNestingLevel,
+		nstack:     []ir.Node{fn},
+		nameFinder: nf,
 	}
-	csa := makeCallSiteAnalyzer(fn, ptab)
+}
+
+// computeCallSiteTable builds and returns a table of call sites for
+// the specified region in function fn. A region here corresponds to a
+// specific subtree within the AST for a function. The main intended
+// use cases are for 'region' to be either A) an entire function body,
+// or B) an inlined call expression.
+func computeCallSiteTable(fn *ir.Func, region ir.Nodes, cstab CallSiteTab, ptab map[ir.Node]pstate, loopNestingLevel int, nf *nameFinder) CallSiteTab {
+	cstb := makeCallSiteTableBuilder(fn, cstab, ptab, loopNestingLevel, nf)
 	var doNode func(ir.Node) bool
 	doNode = func(n ir.Node) bool {
-		csa.nodeVisitPre(n)
+		cstb.nodeVisitPre(n)
 		ir.DoChildren(n, doNode)
-		csa.nodeVisitPost(n)
+		cstb.nodeVisitPost(n)
 		return false
 	}
-	doNode(fn)
-	return csa.cstab
+	for _, n := range region {
+		doNode(n)
+	}
+	return cstb.cstab
 }
 
-func (csa *callSiteAnalyzer) flagsForNode(call *ir.CallExpr) CSPropBits {
+func (cstb *callSiteTableBuilder) flagsForNode(call *ir.CallExpr) CSPropBits {
 	var r CSPropBits
 
 	if debugTrace&debugTraceCalls != 0 {
@@ -59,21 +77,21 @@ func (csa *callSiteAnalyzer) flagsForNode(call *ir.CallExpr) CSPropBits {
 	}
 
 	// Set a bit if this call is within a loop.
-	if csa.loopNest > 0 {
+	if cstb.loopNest > 0 {
 		r |= CallSiteInLoop
 	}
 
 	// Set a bit if the call is within an init function (either
 	// compiler-generated or user-written).
-	if csa.isInit {
+	if cstb.isInit {
 		r |= CallSiteInInitFunc
 	}
 
 	// Decide whether to apply the panic path heuristic. Hack: don't
 	// apply this heuristic in the function "main.main" (mostly just
 	// to avoid annoying users).
-	if !isMainMain(csa.fn) {
-		r = csa.determinePanicPathBits(call, r)
+	if !isMainMain(cstb.fn) {
+		r = cstb.determinePanicPathBits(call, r)
 	}
 
 	return r
@@ -84,15 +102,15 @@ func (csa *callSiteAnalyzer) flagsForNode(call *ir.CallExpr) CSPropBits {
 // panic/exit. Do this by walking back up the node stack to see if we
 // can find either A) an enclosing panic, or B) a statement node that
 // we've determined leads to a panic/exit.
-func (csa *callSiteAnalyzer) determinePanicPathBits(call ir.Node, r CSPropBits) CSPropBits {
-	csa.nstack = append(csa.nstack, call)
+func (cstb *callSiteTableBuilder) determinePanicPathBits(call ir.Node, r CSPropBits) CSPropBits {
+	cstb.nstack = append(cstb.nstack, call)
 	defer func() {
-		csa.nstack = csa.nstack[:len(csa.nstack)-1]
+		cstb.nstack = cstb.nstack[:len(cstb.nstack)-1]
 	}()
 
-	for ri := range csa.nstack[:len(csa.nstack)-1] {
-		i := len(csa.nstack) - ri - 1
-		n := csa.nstack[i]
+	for ri := range cstb.nstack[:len(cstb.nstack)-1] {
+		i := len(cstb.nstack) - ri - 1
+		n := cstb.nstack[i]
 		_, isCallExpr := n.(*ir.CallExpr)
 		_, isStmt := n.(ir.Stmt)
 		if isCallExpr {
@@ -100,7 +118,7 @@ func (csa *callSiteAnalyzer) determinePanicPathBits(call ir.Node, r CSPropBits)
 		}
 
 		if debugTrace&debugTraceCalls != 0 {
-			ps, inps := csa.ptab[n]
+			ps, inps := cstb.ptab[n]
 			fmt.Fprintf(os.Stderr, "=-= callpar %d op=%s ps=%s inptab=%v stmt=%v\n", i, n.Op().String(), ps.String(), inps, isStmt)
 		}
 
@@ -108,7 +126,7 @@ func (csa *callSiteAnalyzer) determinePanicPathBits(call ir.Node, r CSPropBits)
 			r |= CallSiteOnPanicPath
 			break
 		}
-		if v, ok := csa.ptab[n]; ok {
+		if v, ok := cstb.ptab[n]; ok {
 			if v == psCallsPanic {
 				r |= CallSiteOnPanicPath
 				break
@@ -121,133 +139,101 @@ func (csa *callSiteAnalyzer) determinePanicPathBits(call ir.Node, r CSPropBits)
 	return r
 }
 
-func (csa *callSiteAnalyzer) addCallSite(callee *ir.Func, call *ir.CallExpr) {
-	flags := csa.flagsForNode(call)
+// propsForArg returns property bits for a given call argument expression arg.
+func (cstb *callSiteTableBuilder) propsForArg(arg ir.Node) ActualExprPropBits {
+	if cval := cstb.constValue(arg); cval != nil {
+		return ActualExprConstant
+	}
+	if cstb.isConcreteConvIface(arg) {
+		return ActualExprIsConcreteConvIface
+	}
+	fname := cstb.funcName(arg)
+	if fname != nil {
+		if fn := fname.Func; fn != nil && typecheck.HaveInlineBody(fn) {
+			return ActualExprIsInlinableFunc
+		}
+		return ActualExprIsFunc
+	}
+	return 0
+}
+
+// argPropsForCall returns a slice of argument properties for the
+// expressions being passed to the callee in the specific call
+// expression; these will be stored in the CallSite object for a given
+// call and then consulted when scoring. If no arg has any interesting
+// properties we try to save some space and return a nil slice.
+func (cstb *callSiteTableBuilder) argPropsForCall(ce *ir.CallExpr) []ActualExprPropBits {
+	rv := make([]ActualExprPropBits, len(ce.Args))
+	somethingInteresting := false
+	for idx := range ce.Args {
+		argProp := cstb.propsForArg(ce.Args[idx])
+		somethingInteresting = somethingInteresting || (argProp != 0)
+		rv[idx] = argProp
+	}
+	if !somethingInteresting {
+		return nil
+	}
+	return rv
+}
+
+func (cstb *callSiteTableBuilder) addCallSite(callee *ir.Func, call *ir.CallExpr) {
+	flags := cstb.flagsForNode(call)
+	argProps := cstb.argPropsForCall(call)
+	if debugTrace&debugTraceCalls != 0 {
+		fmt.Fprintf(os.Stderr, "=-= props %+v for call %v\n", argProps, call)
+	}
 	// FIXME: maybe bulk-allocate these?
 	cs := &CallSite{
-		Call:   call,
-		Callee: callee,
-		Assign: csa.containingAssignment(call),
-		Flags:  flags,
-		ID:     uint(len(csa.cstab)),
+		Call:     call,
+		Callee:   callee,
+		Assign:   cstb.containingAssignment(call),
+		ArgProps: argProps,
+		Flags:    flags,
+		ID:       uint(len(cstb.cstab)),
 	}
-	if _, ok := csa.cstab[call]; ok {
+	if _, ok := cstb.cstab[call]; ok {
 		fmt.Fprintf(os.Stderr, "*** cstab duplicate entry at: %s\n",
 			fmtFullPos(call.Pos()))
 		fmt.Fprintf(os.Stderr, "*** call: %+v\n", call)
 		panic("bad")
 	}
-	if callee.Inl != nil {
-		// Set initial score for callsite to the cost computed
-		// by CanInline; this score will be refined later based
-		// on heuristics.
-		cs.Score = int(callee.Inl.Cost)
-	}
+	// Set initial score for callsite to the cost computed
+	// by CanInline; this score will be refined later based
+	// on heuristics.
+	cs.Score = int(callee.Inl.Cost)
 
-	csa.cstab[call] = cs
+	if cstb.cstab == nil {
+		cstb.cstab = make(CallSiteTab)
+	}
+	cstb.cstab[call] = cs
 	if debugTrace&debugTraceCalls != 0 {
-		fmt.Fprintf(os.Stderr, "=-= added callsite: callee=%s call=%v\n",
-			callee.Sym().Name, callee)
+		fmt.Fprintf(os.Stderr, "=-= added callsite: caller=%v callee=%v n=%s\n",
+			cstb.fn, callee, fmtFullPos(call.Pos()))
 	}
 }
 
-// ScoreCalls assigns numeric scores to each of the callsites in
-// function 'fn'; the lower the score, the more helpful we think it
-// will be to inline.
-//
-// Unlike a lot of the other inline heuristics machinery, callsite
-// scoring can't be done as part of the CanInline call for a function,
-// due to fact that we may be working on a non-trivial SCC. So for
-// example with this SCC:
-//
-//	func foo(x int) {           func bar(x int, f func()) {
-//	  if x != 0 {                  f()
-//	    bar(x, func(){})           foo(x-1)
-//	  }                         }
-//	}
-//
-// We don't want to perform scoring for the 'foo' call in "bar" until
-// after foo has been analyzed, but it's conceivable that CanInline
-// might visit bar before foo for this SCC.
-func ScoreCalls(fn *ir.Func) {
-	enableDebugTraceIfEnv()
-	defer disableDebugTrace()
-	if debugTrace&debugTraceScoring != 0 {
-		fmt.Fprintf(os.Stderr, "=-= ScoreCalls(%v)\n", ir.FuncName(fn))
-	}
-
-	fih, ok := fpmap[fn]
-	if !ok {
-		// TODO: add an assert/panic here.
-		return
-	}
-
-	resultNameTab := make(map[*ir.Name]resultPropAndCS)
-
-	// Sort callsites to avoid any surprises with non deterministic
-	// map iteration order (this is probably not needed, but here just
-	// in case).
-	csl := make([]*CallSite, 0, len(fih.cstab))
-	for _, cs := range fih.cstab {
-		csl = append(csl, cs)
-	}
-	sort.Slice(csl, func(i, j int) bool {
-		return csl[i].ID < csl[j].ID
-	})
-
-	// Score each call site.
-	for _, cs := range csl {
-		var cprops *FuncProps
-		fihcprops := false
-		desercprops := false
-		if fih, ok := fpmap[cs.Callee]; ok {
-			cprops = fih.props
-			fihcprops = true
-		} else if cs.Callee.Inl != nil {
-			cprops = DeserializeFromString(cs.Callee.Inl.Properties)
-			desercprops = true
-		} else {
-			if base.Debug.DumpInlFuncProps != "" {
-				fmt.Fprintf(os.Stderr, "=-= *** unable to score call to %s from %s\n", cs.Callee.Sym().Name, fmtFullPos(cs.Call.Pos()))
-				panic("should never happen")
-			} else {
-				continue
-			}
-		}
-		cs.Score, cs.ScoreMask = computeCallSiteScore(cs.Callee, cprops, cs.Call, cs.Flags)
-
-		examineCallResults(cs, resultNameTab)
-
-		if debugTrace&debugTraceScoring != 0 {
-			fmt.Fprintf(os.Stderr, "=-= scoring call at %s: flags=%d score=%d fih=%v deser=%v\n", fmtFullPos(cs.Call.Pos()), cs.Flags, cs.Score, fihcprops, desercprops)
-		}
-	}
-
-	rescoreBasedOnCallResultUses(fn, resultNameTab, fih.cstab)
-}
-
-func (csa *callSiteAnalyzer) nodeVisitPre(n ir.Node) {
+func (cstb *callSiteTableBuilder) nodeVisitPre(n ir.Node) {
 	switch n.Op() {
 	case ir.ORANGE, ir.OFOR:
 		if !hasTopLevelLoopBodyReturnOrBreak(loopBody(n)) {
-			csa.loopNest++
+			cstb.loopNest++
 		}
 	case ir.OCALLFUNC:
 		ce := n.(*ir.CallExpr)
 		callee := pgo.DirectCallee(ce.Fun)
 		if callee != nil && callee.Inl != nil {
-			csa.addCallSite(callee, ce)
+			cstb.addCallSite(callee, ce)
 		}
 	}
-	csa.nstack = append(csa.nstack, n)
+	cstb.nstack = append(cstb.nstack, n)
 }
 
-func (csa *callSiteAnalyzer) nodeVisitPost(n ir.Node) {
-	csa.nstack = csa.nstack[:len(csa.nstack)-1]
+func (cstb *callSiteTableBuilder) nodeVisitPost(n ir.Node) {
+	cstb.nstack = cstb.nstack[:len(cstb.nstack)-1]
 	switch n.Op() {
 	case ir.ORANGE, ir.OFOR:
 		if !hasTopLevelLoopBodyReturnOrBreak(loopBody(n)) {
-			csa.loopNest--
+			cstb.loopNest--
 		}
 	}
 }
@@ -308,8 +294,8 @@ func hasTopLevelLoopBodyReturnOrBreak(loopBody ir.Nodes) bool {
 // call to a pair of auto-temps, then the second one assigning the
 // auto-temps to the user-visible vars. This helper will return the
 // second (outer) of these two.
-func (csa *callSiteAnalyzer) containingAssignment(n ir.Node) ir.Node {
-	parent := csa.nstack[len(csa.nstack)-1]
+func (cstb *callSiteTableBuilder) containingAssignment(n ir.Node) ir.Node {
+	parent := cstb.nstack[len(cstb.nstack)-1]
 
 	// assignsOnlyAutoTemps returns TRUE of the specified OAS2FUNC
 	// node assigns only auto-temps.
@@ -342,12 +328,12 @@ func (csa *callSiteAnalyzer) containingAssignment(n ir.Node) ir.Node {
 		// OAS1({x,y},OCONVNOP(OAS2FUNC({auto1,auto2},OCALLFUNC(bar))))
 		//
 		if assignsOnlyAutoTemps(parent) {
-			par2 := csa.nstack[len(csa.nstack)-2]
+			par2 := cstb.nstack[len(cstb.nstack)-2]
 			if par2.Op() == ir.OAS2 {
 				return par2
 			}
 			if par2.Op() == ir.OCONVNOP {
-				par3 := csa.nstack[len(csa.nstack)-3]
+				par3 := cstb.nstack[len(cstb.nstack)-3]
 				if par3.Op() == ir.OAS2 {
 					return par3
 				}
@@ -357,3 +343,71 @@ func (csa *callSiteAnalyzer) containingAssignment(n ir.Node) ir.Node {
 
 	return nil
 }
+
+// UpdateCallsiteTable handles updating of callerfn's call site table
+// after an inlined has been carried out, e.g. the call at 'n' as been
+// turned into the inlined call expression 'ic' within function
+// callerfn. The chief thing of interest here is to make sure that any
+// call nodes within 'ic' are added to the call site table for
+// 'callerfn' and scored appropriately.
+func UpdateCallsiteTable(callerfn *ir.Func, n *ir.CallExpr, ic *ir.InlinedCallExpr) {
+	enableDebugTraceIfEnv()
+	defer disableDebugTrace()
+
+	funcInlHeur, ok := fpmap[callerfn]
+	if !ok {
+		// This can happen for compiler-generated wrappers.
+		if debugTrace&debugTraceCalls != 0 {
+			fmt.Fprintf(os.Stderr, "=-= early exit, no entry for caller fn %v\n", callerfn)
+		}
+		return
+	}
+
+	if debugTrace&debugTraceCalls != 0 {
+		fmt.Fprintf(os.Stderr, "=-= UpdateCallsiteTable(caller=%v, cs=%s)\n",
+			callerfn, fmtFullPos(n.Pos()))
+	}
+
+	// Mark the call in question as inlined.
+	oldcs, ok := funcInlHeur.cstab[n]
+	if !ok {
+		// This can happen for compiler-generated wrappers.
+		return
+	}
+	oldcs.aux |= csAuxInlined
+
+	if debugTrace&debugTraceCalls != 0 {
+		fmt.Fprintf(os.Stderr, "=-= marked as inlined: callee=%v %s\n",
+			oldcs.Callee, EncodeCallSiteKey(oldcs))
+	}
+
+	// Walk the inlined call region to collect new callsites.
+	var icp pstate
+	if oldcs.Flags&CallSiteOnPanicPath != 0 {
+		icp = psCallsPanic
+	}
+	var loopNestLevel int
+	if oldcs.Flags&CallSiteInLoop != 0 {
+		loopNestLevel = 1
+	}
+	ptab := map[ir.Node]pstate{ic: icp}
+	nf := newNameFinder(nil)
+	icstab := computeCallSiteTable(callerfn, ic.Body, nil, ptab, loopNestLevel, nf)
+
+	// Record parent callsite. This is primarily for debug output.
+	for _, cs := range icstab {
+		cs.parent = oldcs
+	}
+
+	// Score the calls in the inlined body. Note the setting of
+	// "doCallResults" to false here: at the moment there isn't any
+	// easy way to localize or region-ize the work done by
+	// "rescoreBasedOnCallResultUses", which currently does a walk
+	// over the entire function to look for uses of a given set of
+	// results. Similarly we're passing nil to makeCallSiteAnalyzer,
+	// so as to run name finding without the use of static value &
+	// friends.
+	csa := makeCallSiteAnalyzer(nil)
+	const doCallResults = false
+	csa.scoreCallsRegion(callerfn, ic.Body, icstab, doCallResults, ic)
+}

src/cmd/compile/internal/inline/inlheur/analyze_func_flags.go

@@ -40,8 +40,8 @@ func makeFuncFlagsAnalyzer(fn *ir.Func) *funcFlagsAnalyzer {
 	}
 }
 
-// setResults transfers func flag results to 'fp'.
-func (ffa *funcFlagsAnalyzer) setResults(fp *FuncProps) {
+// setResults transfers func flag results to 'funcProps'.
+func (ffa *funcFlagsAnalyzer) setResults(funcProps *FuncProps) {
 	var rv FuncPropBits
 	if !ffa.noInfo && ffa.stateForList(ffa.fn.Body) == psCallsPanic {
 		rv = FuncPropNeverReturns
@@ -63,37 +63,27 @@ func (ffa *funcFlagsAnalyzer) setResults(fp *FuncProps) {
 	if isMainMain(ffa.fn) {
 		rv &^= FuncPropNeverReturns
 	}
-	fp.Flags = rv
+	funcProps.Flags = rv
 }
 
-func (ffa *funcFlagsAnalyzer) getstate(n ir.Node) pstate {
-	val, ok := ffa.nstate[n]
-	if !ok {
-		base.Fatalf("funcFlagsAnalyzer: fn %q node %s line %s: internal error, no setting for node:\n%+v\n", ffa.fn.Sym().Name, n.Op().String(), ir.Line(n), n)
-	}
-	return val
+func (ffa *funcFlagsAnalyzer) getState(n ir.Node) pstate {
+	return ffa.nstate[n]
 }
 
-func (ffa *funcFlagsAnalyzer) setstate(n ir.Node, st pstate) {
-	if _, ok := ffa.nstate[n]; ok {
-		base.Fatalf("funcFlagsAnalyzer: fn %q internal error, existing setting for node:\n%+v\n", ffa.fn.Sym().Name, n)
-	} else {
+func (ffa *funcFlagsAnalyzer) setState(n ir.Node, st pstate) {
+	if st != psNoInfo {
 		ffa.nstate[n] = st
 	}
 }
 
-func (ffa *funcFlagsAnalyzer) updatestate(n ir.Node, st pstate) {
-	if _, ok := ffa.nstate[n]; !ok {
-		base.Fatalf("funcFlagsAnalyzer: fn %q internal error, expected existing setting for node:\n%+v\n", ffa.fn.Sym().Name, n)
+func (ffa *funcFlagsAnalyzer) updateState(n ir.Node, st pstate) {
+	if st == psNoInfo {
+		delete(ffa.nstate, n)
 	} else {
 		ffa.nstate[n] = st
 	}
 }
 
-func (ffa *funcFlagsAnalyzer) setstateSoft(n ir.Node, st pstate) {
-	ffa.nstate[n] = st
-}
-
 func (ffa *funcFlagsAnalyzer) panicPathTable() map[ir.Node]pstate {
 	return ffa.nstate
 }
@@ -148,15 +138,29 @@ func branchCombine(p1, p2 pstate) pstate {
 // as updating disposition of intermediate nodes.
 func (ffa *funcFlagsAnalyzer) stateForList(list ir.Nodes) pstate {
 	st := psTop
-	for i := range list {
+	// Walk the list backwards so that we can update the state for
+	// earlier list elements based on what we find out about their
+	// successors. Example:
+	//
+	//        if ... {
+	//  L10:    foo()
+	//  L11:    <stmt>
+	//  L12:    panic(...)
+	//        }
+	//
+	// After combining the dispositions for line 11 and 12, we want to
+	// update the state for the call at line 10 based on that combined
+	// disposition (if L11 has no path to "return", then the call at
+	// line 10 will be on a panic path).
+	for i := len(list) - 1; i >= 0; i-- {
 		n := list[i]
-		psi := ffa.getstate(n)
+		psi := ffa.getState(n)
 		if debugTrace&debugTraceFuncFlags != 0 {
 			fmt.Fprintf(os.Stderr, "=-= %v: stateForList n=%s ps=%s\n",
 				ir.Line(n), n.Op().String(), psi.String())
 		}
-		st = blockCombine(st, psi)
-		ffa.updatestate(n, st)
+		st = blockCombine(psi, st)
+		ffa.updateState(n, st)
 	}
 	if st == psTop {
 		st = psNoInfo
@@ -189,8 +193,8 @@ func isExitCall(n ir.Node) bool {
 		isWellKnownFunc(s, "runtime", "throw") {
 		return true
 	}
-	if fp := propsForFunc(name.Func); fp != nil {
-		if fp.Flags&FuncPropNeverReturns != 0 {
+	if funcProps := propsForFunc(name.Func); funcProps != nil {
+		if funcProps.Flags&FuncPropNeverReturns != 0 {
 			return true
 		}
 	}
@@ -223,8 +227,6 @@ func (ffa *funcFlagsAnalyzer) nodeVisitPost(n ir.Node) {
 			ir.Line(n), n.Op().String(), shouldVisit(n))
 	}
 	if !shouldVisit(n) {
-		// invoke soft set, since node may be shared (e.g. ONAME)
-		ffa.setstateSoft(n, psNoInfo)
 		return
 	}
 	var st pstate
@@ -347,7 +349,7 @@ func (ffa *funcFlagsAnalyzer) nodeVisitPost(n ir.Node) {
 		fmt.Fprintf(os.Stderr, "=-= %v: visit n=%s returns %s\n",
 			ir.Line(n), n.Op().String(), st.String())
 	}
-	ffa.setstate(n, st)
+	ffa.setState(n, st)
 }
 
 func (ffa *funcFlagsAnalyzer) nodeVisitPre(n ir.Node) {

src/cmd/compile/internal/inline/inlheur/analyze_func_params.go

@@ -19,6 +19,7 @@ type paramsAnalyzer struct {
 	params []*ir.Name
 	top    []bool
 	*condLevelTracker
+	*nameFinder
 }
 
 // getParams returns an *ir.Name slice containing all params for the
@@ -29,10 +30,36 @@ func getParams(fn *ir.Func) []*ir.Name {
 	return fn.Dcl[:numParams]
 }
 
-func makeParamsAnalyzer(fn *ir.Func) *paramsAnalyzer {
+// addParamsAnalyzer creates a new paramsAnalyzer helper object for
+// the function fn, appends it to the analyzers list, and returns the
+// new list. If the function in question doesn't have any interesting
+// parameters then the analyzer list is returned unchanged, and the
+// params flags in "fp" are updated accordingly.
+func addParamsAnalyzer(fn *ir.Func, analyzers []propAnalyzer, fp *FuncProps, nf *nameFinder) []propAnalyzer {
+	pa, props := makeParamsAnalyzer(fn, nf)
+	if pa != nil {
+		analyzers = append(analyzers, pa)
+	} else {
+		fp.ParamFlags = props
+	}
+	return analyzers
+}
+
+// makeParamAnalyzer creates a new helper object to analyze parameters
+// of function fn. If the function doesn't have any interesting
+// params, a nil helper is returned along with a set of default param
+// flags for the func.
+func makeParamsAnalyzer(fn *ir.Func, nf *nameFinder) (*paramsAnalyzer, []ParamPropBits) {
 	params := getParams(fn) // includes receiver if applicable
+	if len(params) == 0 {
+		return nil, nil
+	}
 	vals := make([]ParamPropBits, len(params))
+	if fn.Inl == nil {
+		return nil, vals
+	}
 	top := make([]bool, len(params))
+	interestingToAnalyze := false
 	for i, pn := range params {
 		if pn == nil {
 			continue
@@ -48,6 +75,10 @@ func makeParamsAnalyzer(fn *ir.Func) *paramsAnalyzer {
 			continue
 		}
 		top[i] = true
+		interestingToAnalyze = true
+	}
+	if !interestingToAnalyze {
+		return nil, vals
 	}
 
 	if debugTrace&debugTraceParams != 0 {
@@ -58,22 +89,23 @@ func makeParamsAnalyzer(fn *ir.Func) *paramsAnalyzer {
 			if params[i] != nil {
 				n = params[i].Sym().String()
 			}
-			fmt.Fprintf(os.Stderr, "=-=  %d: %q %s\n",
-				i, n, vals[i].String())
+			fmt.Fprintf(os.Stderr, "=-=  %d: %q %s top=%v\n",
+				i, n, vals[i].String(), top[i])
 		}
 	}
-
-	return &paramsAnalyzer{
+	pa := &paramsAnalyzer{
 		fname:            fn.Sym().Name,
 		values:           vals,
 		params:           params,
 		top:              top,
 		condLevelTracker: new(condLevelTracker),
+		nameFinder:       nf,
 	}
+	return pa, nil
 }
 
-func (pa *paramsAnalyzer) setResults(fp *FuncProps) {
-	fp.ParamFlags = pa.values
+func (pa *paramsAnalyzer) setResults(funcProps *FuncProps) {
+	funcProps.ParamFlags = pa.values
 }
 
 func (pa *paramsAnalyzer) findParamIdx(n *ir.Name) int {
@@ -132,7 +164,7 @@ func (pa *paramsAnalyzer) callCheckParams(ce *ir.CallExpr) {
 			return
 		}
 		sel := ce.Fun.(*ir.SelectorExpr)
-		r := ir.StaticValue(sel.X)
+		r := pa.staticValue(sel.X)
 		if r.Op() != ir.ONAME {
 			return
 		}
@@ -163,8 +195,8 @@ func (pa *paramsAnalyzer) callCheckParams(ce *ir.CallExpr) {
 					return name == p, false
 				})
 		} else {
-			cname, isFunc, _ := isFuncName(called)
-			if isFunc {
+			cname := pa.funcName(called)
+			if cname != nil {
 				pa.deriveFlagsFromCallee(ce, cname.Func)
 			}
 		}
@@ -208,7 +240,7 @@ func (pa *paramsAnalyzer) deriveFlagsFromCallee(ce *ir.CallExpr, callee *ir.Func
 		}
 		// See if one of the caller's parameters is flowing unmodified
 		// into this actual expression.
-		r := ir.StaticValue(arg)
+		r := pa.staticValue(arg)
 		if r.Op() != ir.ONAME {
 			return
 		}
@@ -217,7 +249,13 @@ func (pa *paramsAnalyzer) deriveFlagsFromCallee(ce *ir.CallExpr, callee *ir.Func
 			return
 		}
 		callerParamIdx := pa.findParamIdx(name)
-		if callerParamIdx == -1 || pa.params[callerParamIdx] == nil {
+		// note that callerParamIdx may return -1 in the case where
+		// the param belongs not to the current closure func we're
+		// analyzing but to an outer enclosing func.
+		if callerParamIdx == -1 {
+			return
+		}
+		if pa.params[callerParamIdx] == nil {
 			panic("something went wrong")
 		}
 		if !pa.top[callerParamIdx] &&

src/cmd/compile/internal/inline/inlheur/analyze_func_returns.go

@@ -12,14 +12,15 @@ import (
 	"os"
 )
 
-// returnsAnalyzer stores state information for the process of
+// resultsAnalyzer stores state information for the process of
 // computing flags/properties for the return values of a specific Go
 // function, as part of inline heuristics synthesis.
-type returnsAnalyzer struct {
-	fname     string
-	props     []ResultPropBits
-	values    []resultVal
-	canInline func(*ir.Func)
+type resultsAnalyzer struct {
+	fname           string
+	props           []ResultPropBits
+	values          []resultVal
+	inlineMaxBudget int
+	*nameFinder
 }
 
 // resultVal captures information about a specific result returned from
@@ -28,70 +29,99 @@ type returnsAnalyzer struct {
 // the same function, etc. This container stores info on a the specific
 // scenarios we're looking for.
 type resultVal struct {
-	lit     constant.Value
+	cval    constant.Value
 	fn      *ir.Name
 	fnClo   bool
 	top     bool
 	derived bool // see deriveReturnFlagsFromCallee below
 }
 
-func makeResultsAnalyzer(fn *ir.Func, canInline func(*ir.Func)) *returnsAnalyzer {
+// addResultsAnalyzer creates a new resultsAnalyzer helper object for
+// the function fn, appends it to the analyzers list, and returns the
+// new list. If the function in question doesn't have any returns (or
+// any interesting returns) then the analyzer list is left as is, and
+// the result flags in "fp" are updated accordingly.
+func addResultsAnalyzer(fn *ir.Func, analyzers []propAnalyzer, fp *FuncProps, inlineMaxBudget int, nf *nameFinder) []propAnalyzer {
+	ra, props := makeResultsAnalyzer(fn, inlineMaxBudget, nf)
+	if ra != nil {
+		analyzers = append(analyzers, ra)
+	} else {
+		fp.ResultFlags = props
+	}
+	return analyzers
+}
+
+// makeResultsAnalyzer creates a new helper object to analyze results
+// in function fn. If the function doesn't have any interesting
+// results, a nil helper is returned along with a set of default
+// result flags for the func.
+func makeResultsAnalyzer(fn *ir.Func, inlineMaxBudget int, nf *nameFinder) (*resultsAnalyzer, []ResultPropBits) {
 	results := fn.Type().Results()
+	if len(results) == 0 {
+		return nil, nil
+	}
 	props := make([]ResultPropBits, len(results))
+	if fn.Inl == nil {
+		return nil, props
+	}
 	vals := make([]resultVal, len(results))
+	interestingToAnalyze := false
 	for i := range results {
 		rt := results[i].Type
 		if !rt.IsScalar() && !rt.HasNil() {
 			// existing properties not applicable here (for things
 			// like structs, arrays, slices, etc).
-			props[i] = ResultNoInfo
 			continue
 		}
 		// set the "top" flag (as in "top element of data flow lattice")
 		// meaning "we have no info yet, but we might later on".
 		vals[i].top = true
+		interestingToAnalyze = true
 	}
-	return &returnsAnalyzer{
-		props:     props,
-		values:    vals,
-		canInline: canInline,
+	if !interestingToAnalyze {
+		return nil, props
 	}
+	ra := &resultsAnalyzer{
+		props:           props,
+		values:          vals,
+		inlineMaxBudget: inlineMaxBudget,
+		nameFinder:      nf,
+	}
+	return ra, nil
 }
 
 // setResults transfers the calculated result properties for this
-// function to 'fp'.
-func (ra *returnsAnalyzer) setResults(fp *FuncProps) {
+// function to 'funcProps'.
+func (ra *resultsAnalyzer) setResults(funcProps *FuncProps) {
 	// Promote ResultAlwaysSameFunc to ResultAlwaysSameInlinableFunc
 	for i := range ra.values {
 		if ra.props[i] == ResultAlwaysSameFunc && !ra.values[i].derived {
 			f := ra.values[i].fn.Func
-			// If the function being returns is a closure that hasn't
-			// yet been checked by CanInline, invoke it now. NB: this
-			// is hacky, it would be better if things were structured
-			// so that all closures were visited ahead of time.
-			if ra.values[i].fnClo {
-				if f != nil && !f.InlinabilityChecked() {
-					ra.canInline(f)
-				}
-			}
-			if f.Inl != nil {
+			// HACK: in order to allow for call site score
+			// adjustments, we used a relaxed inline budget in
+			// determining inlinability. For the check below, however,
+			// we want to know is whether the func in question is
+			// likely to be inlined, as opposed to whether it might
+			// possibly be inlined if all the right score adjustments
+			// happened, so do a simple check based on the cost.
+			if f.Inl != nil && f.Inl.Cost <= int32(ra.inlineMaxBudget) {
 				ra.props[i] = ResultAlwaysSameInlinableFunc
 			}
 		}
 	}
-	fp.ResultFlags = ra.props
+	funcProps.ResultFlags = ra.props
 }
 
-func (ra *returnsAnalyzer) pessimize() {
+func (ra *resultsAnalyzer) pessimize() {
 	for i := range ra.props {
 		ra.props[i] = ResultNoInfo
 	}
 }
 
-func (ra *returnsAnalyzer) nodeVisitPre(n ir.Node) {
+func (ra *resultsAnalyzer) nodeVisitPre(n ir.Node) {
 }
 
-func (ra *returnsAnalyzer) nodeVisitPost(n ir.Node) {
+func (ra *resultsAnalyzer) nodeVisitPost(n ir.Node) {
 	if len(ra.values) == 0 {
 		return
 	}
@@ -115,48 +145,29 @@ func (ra *returnsAnalyzer) nodeVisitPost(n ir.Node) {
 	}
 }
 
-// isFuncName returns the *ir.Name for the func or method
-// corresponding to node 'n', along with a boolean indicating success,
-// and another boolean indicating whether the func is closure.
-func isFuncName(n ir.Node) (*ir.Name, bool, bool) {
-	sv := ir.StaticValue(n)
-	if sv.Op() == ir.ONAME {
-		name := sv.(*ir.Name)
-		if name.Sym() != nil && name.Class == ir.PFUNC {
-			return name, true, false
-		}
-	}
-	if sv.Op() == ir.OCLOSURE {
-		cloex := sv.(*ir.ClosureExpr)
-		return cloex.Func.Nname, true, true
-	}
-	if sv.Op() == ir.OMETHEXPR {
-		if mn := ir.MethodExprName(sv); mn != nil {
-			return mn, true, false
-		}
-	}
-	return nil, false, false
-}
-
 // analyzeResult examines the expression 'n' being returned as the
 // 'ii'th argument in some return statement to see whether has
 // interesting characteristics (for example, returns a constant), then
 // applies a dataflow "meet" operation to combine this result with any
 // previous result (for the given return slot) that we've already
 // processed.
-func (ra *returnsAnalyzer) analyzeResult(ii int, n ir.Node) {
-	isAllocMem := isAllocatedMem(n)
-	isConcConvItf := isConcreteConvIface(n)
-	lit, isConst := isLiteral(n)
-	rfunc, isFunc, isClo := isFuncName(n)
+func (ra *resultsAnalyzer) analyzeResult(ii int, n ir.Node) {
+	isAllocMem := ra.isAllocatedMem(n)
+	isConcConvItf := ra.isConcreteConvIface(n)
+	constVal := ra.constValue(n)
+	isConst := (constVal != nil)
+	isNil := ra.isNil(n)
+	rfunc := ra.funcName(n)
+	isFunc := (rfunc != nil)
+	isClo := (rfunc != nil && rfunc.Func.OClosure != nil)
 	curp := ra.props[ii]
-	dprops, isDerivedFromCall := deriveReturnFlagsFromCallee(n)
+	dprops, isDerivedFromCall := ra.deriveReturnFlagsFromCallee(n)
 	newp := ResultNoInfo
-	var newlit constant.Value
+	var newcval constant.Value
 	var newfunc *ir.Name
 
 	if debugTrace&debugTraceResults != 0 {
-		fmt.Fprintf(os.Stderr, "=-= %v: analyzeResult n=%s ismem=%v isconcconv=%v isconst=%v isfunc=%v isclo=%v\n", ir.Line(n), n.Op().String(), isAllocMem, isConcConvItf, isConst, isFunc, isClo)
+		fmt.Fprintf(os.Stderr, "=-= %v: analyzeResult n=%s ismem=%v isconcconv=%v isconst=%v isnil=%v isfunc=%v isclo=%v\n", ir.Line(n), n.Op().String(), isAllocMem, isConcConvItf, isConst, isNil, isFunc, isClo)
 	}
 
 	if ra.values[ii].top {
@@ -173,7 +184,10 @@ func (ra *returnsAnalyzer) analyzeResult(ii int, n ir.Node) {
 			newfunc = rfunc
 		case isConst:
 			newp = ResultAlwaysSameConstant
-			newlit = lit
+			newcval = constVal
+		case isNil:
+			newp = ResultAlwaysSameConstant
+			newcval = nil
 		case isDerivedFromCall:
 			newp = dprops
 			ra.values[ii].derived = true
@@ -186,17 +200,20 @@ func (ra *returnsAnalyzer) analyzeResult(ii int, n ir.Node) {
 			// the previous returns.
 			switch curp {
 			case ResultIsAllocatedMem:
-				if isAllocatedMem(n) {
+				if isAllocMem {
 					newp = ResultIsAllocatedMem
 				}
 			case ResultIsConcreteTypeConvertedToInterface:
-				if isConcreteConvIface(n) {
+				if isConcConvItf {
 					newp = ResultIsConcreteTypeConvertedToInterface
 				}
 			case ResultAlwaysSameConstant:
-				if isConst && isSameLiteral(lit, ra.values[ii].lit) {
+				if isNil && ra.values[ii].cval == nil {
 					newp = ResultAlwaysSameConstant
-					newlit = lit
+					newcval = nil
+				} else if isConst && constant.Compare(constVal, token.EQL, ra.values[ii].cval) {
+					newp = ResultAlwaysSameConstant
+					newcval = constVal
 				}
 			case ResultAlwaysSameFunc:
 				if isFunc && isSameFuncName(rfunc, ra.values[ii].fn) {
@@ -208,7 +225,7 @@ func (ra *returnsAnalyzer) analyzeResult(ii int, n ir.Node) {
 	}
 	ra.values[ii].fn = newfunc
 	ra.values[ii].fnClo = isClo
-	ra.values[ii].lit = newlit
+	ra.values[ii].cval = newcval
 	ra.props[ii] = newp
 
 	if debugTrace&debugTraceResults != 0 {
@@ -217,15 +234,6 @@ func (ra *returnsAnalyzer) analyzeResult(ii int, n ir.Node) {
 	}
 }
 
-func isAllocatedMem(n ir.Node) bool {
-	sv := ir.StaticValue(n)
-	switch sv.Op() {
-	case ir.OMAKESLICE, ir.ONEW, ir.OPTRLIT, ir.OSLICELIT:
-		return true
-	}
-	return false
-}
-
 // deriveReturnFlagsFromCallee tries to set properties for a given
 // return result where we're returning call expression; return value
 // is a return property value and a boolean indicating whether the
@@ -242,7 +250,7 @@ func isAllocatedMem(n ir.Node) bool {
 // set foo's return property to that of bar. In the case of "two", however,
 // even though each return path returns a constant, we don't know
 // whether the constants are identical, hence we need to be conservative.
-func deriveReturnFlagsFromCallee(n ir.Node) (ResultPropBits, bool) {
+func (ra *resultsAnalyzer) deriveReturnFlagsFromCallee(n ir.Node) (ResultPropBits, bool) {
 	if n.Op() != ir.OCALLFUNC {
 		return 0, false
 	}
@@ -254,8 +262,8 @@ func deriveReturnFlagsFromCallee(n ir.Node) (ResultPropBits, bool) {
 	if called.Op() != ir.ONAME {
 		return 0, false
 	}
-	cname, isFunc, _ := isFuncName(called)
-	if !isFunc {
+	cname := ra.funcName(called)
+	if cname == nil {
 		return 0, false
 	}
 	calleeProps := propsForFunc(cname.Func)
@@ -267,41 +275,3 @@ func deriveReturnFlagsFromCallee(n ir.Node) (ResultPropBits, bool) {
 	}
 	return calleeProps.ResultFlags[0], true
 }
-
-func isLiteral(n ir.Node) (constant.Value, bool) {
-	sv := ir.StaticValue(n)
-	switch sv.Op() {
-	case ir.ONIL:
-		return nil, true
-	case ir.OLITERAL:
-		return sv.Val(), true
-	}
-	return nil, false
-}
-
-// isSameLiteral checks to see if 'v1' and 'v2' correspond to the same
-// literal value, or if they are both nil.
-func isSameLiteral(v1, v2 constant.Value) bool {
-	if v1 == nil && v2 == nil {
-		return true
-	}
-	if v1 == nil || v2 == nil {
-		return false
-	}
-	return constant.Compare(v1, token.EQL, v2)
-}
-
-func isConcreteConvIface(n ir.Node) bool {
-	sv := ir.StaticValue(n)
-	if sv.Op() != ir.OCONVIFACE {
-		return false
-	}
-	return !sv.(*ir.ConvExpr).X.Type().IsInterface()
-}
-
-func isSameFuncName(v1, v2 *ir.Name) bool {
-	// NB: there are a few corner cases where pointer equality
-	// doesn't work here, but this should be good enough for
-	// our purposes here.
-	return v1 == v2
-}

src/cmd/compile/internal/inline/inlheur/callsite.go

@@ -25,13 +25,17 @@ import (
 // the site, and "ID" is a numeric ID for the site within its
 // containing function.
 type CallSite struct {
-	Callee    *ir.Func
-	Call      *ir.CallExpr
-	Assign    ir.Node
-	Flags     CSPropBits
+	Callee *ir.Func
+	Call   *ir.CallExpr
+	parent *CallSite
+	Assign ir.Node
+	Flags  CSPropBits
+
+	ArgProps  []ActualExprPropBits
 	Score     int
 	ScoreMask scoreAdjustTyp
 	ID        uint
+	aux       uint8
 }
 
 // CallSiteTab is a table of call sites, keyed by call expr.
@@ -41,20 +45,16 @@ type CallSite struct {
 // with many calls that share the same auto-generated pos.
 type CallSiteTab map[*ir.CallExpr]*CallSite
 
-// Package-level table of callsites.
-var cstab = CallSiteTab{}
+// ActualExprPropBits describes a property of an actual expression (value
+// passed to some specific func argument at a call site).
+type ActualExprPropBits uint8
 
-func GetCallSiteScore(ce *ir.CallExpr) (bool, int) {
-	cs, ok := cstab[ce]
-	if !ok {
-		return false, 0
-	}
-	return true, cs.Score
-}
-
-func CallSiteTable() CallSiteTab {
-	return cstab
-}
+const (
+	ActualExprConstant ActualExprPropBits = 1 << iota
+	ActualExprIsConcreteConvIface
+	ActualExprIsFunc
+	ActualExprIsInlinableFunc
+)
 
 type CSPropBits uint32
 
@@ -64,6 +64,12 @@ const (
 	CallSiteInInitFunc
 )
 
+type csAuxBits uint8
+
+const (
+	csAuxInlined = 1 << iota
+)
+
 // encodedCallSiteTab is a table keyed by "encoded" callsite
 // (stringified src.XPos plus call site ID) mapping to a value of call
 // property bits and score.

src/cmd/compile/internal/inline/inlheur/debugflags_test.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package inlheur
+
+import (
+	"testing"
+)
+
+func TestInlScoreAdjFlagParse(t *testing.T) {
+	scenarios := []struct {
+		value string
+		expok bool
+	}{
+		{
+			value: "returnFeedsConcreteToInterfaceCallAdj:9",
+			expok: true,
+		},
+		{
+			value: "panicPathAdj:-1/initFuncAdj:9",
+			expok: true,
+		},
+		{
+			value: "",
+			expok: false,
+		},
+		{
+			value: "nonsenseAdj:10",
+			expok: false,
+		},
+		{
+			value: "inLoopAdj:",
+			expok: false,
+		},
+		{
+			value: "inLoopAdj:10:10",
+			expok: false,
+		},
+		{
+			value: "inLoopAdj:blah",
+			expok: false,
+		},
+		{
+			value: "/",
+			expok: false,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		err := parseScoreAdj(scenario.value)
+		t.Logf("for value=%q err is %v\n", scenario.value, err)
+		if scenario.expok {
+			if err != nil {
+				t.Errorf("expected parseScoreAdj(%s) ok, got err %v",
+					scenario.value, err)
+			}
+		} else {
+			if err == nil {
+				t.Errorf("expected parseScoreAdj(%s) failure, got success",
+					scenario.value)
+			}
+		}
+	}
+}

src/cmd/compile/internal/inline/inlheur/dumpscores_test.go

@@ -0,0 +1,109 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package inlheur
+
+import (
+	"internal/testenv"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestDumpCallSiteScoreDump(t *testing.T) {
+	td := t.TempDir()
+	testenv.MustHaveGoBuild(t)
+
+	scenarios := []struct {
+		name               string
+		promoted           int
+		indirectlyPromoted int
+		demoted            int
+		unchanged          int
+	}{
+		{
+			name:               "dumpscores",
+			promoted:           1,
+			indirectlyPromoted: 1,
+			demoted:            1,
+			unchanged:          5,
+		},
+	}
+
+	for _, scen := range scenarios {
+		dumpfile, err := gatherInlCallSitesScoresForFile(t, scen.name, td)
+		if err != nil {
+			t.Fatalf("dumping callsite scores for %q: error %v", scen.name, err)
+		}
+		var lines []string
+		if content, err := os.ReadFile(dumpfile); err != nil {
+			t.Fatalf("reading dump %q: error %v", dumpfile, err)
+		} else {
+			lines = strings.Split(string(content), "\n")
+		}
+		prom, indprom, dem, unch := 0, 0, 0, 0
+		for _, line := range lines {
+			switch {
+			case strings.TrimSpace(line) == "":
+			case !strings.Contains(line, "|"):
+			case strings.HasPrefix(line, "#"):
+			case strings.Contains(line, "PROMOTED"):
+				prom++
+			case strings.Contains(line, "INDPROM"):
+				indprom++
+			case strings.Contains(line, "DEMOTED"):
+				dem++
+			default:
+				unch++
+			}
+		}
+		showout := false
+		if prom != scen.promoted {
+			t.Errorf("testcase %q, got %d promoted want %d promoted",
+				scen.name, prom, scen.promoted)
+			showout = true
+		}
+		if indprom != scen.indirectlyPromoted {
+			t.Errorf("testcase %q, got %d indirectly promoted want %d",
+				scen.name, indprom, scen.indirectlyPromoted)
+			showout = true
+		}
+		if dem != scen.demoted {
+			t.Errorf("testcase %q, got %d demoted want %d demoted",
+				scen.name, dem, scen.demoted)
+			showout = true
+		}
+		if unch != scen.unchanged {
+			t.Errorf("testcase %q, got %d unchanged want %d unchanged",
+				scen.name, unch, scen.unchanged)
+			showout = true
+		}
+		if showout {
+			t.Logf(">> dump output: %s", strings.Join(lines, "\n"))
+		}
+	}
+}
+
+// gatherInlCallSitesScoresForFile builds the specified testcase 'testcase'
+// from testdata/props passing the "-d=dumpinlcallsitescores=1"
+// compiler option, to produce a dump, then returns the path of the
+// newly created file.
+func gatherInlCallSitesScoresForFile(t *testing.T, testcase string, td string) (string, error) {
+	t.Helper()
+	gopath := "testdata/" + testcase + ".go"
+	outpath := filepath.Join(td, testcase+".a")
+	dumpfile := filepath.Join(td, testcase+".callsites.txt")
+	run := []string{testenv.GoToolPath(t), "build",
+		"-gcflags=-d=dumpinlcallsitescores=1", "-o", outpath, gopath}
+	out, err := testenv.Command(t, run[0], run[1:]...).CombinedOutput()
+	t.Logf("run: %+v\n", run)
+	if err != nil {
+		return "", err
+	}
+	if err := os.WriteFile(dumpfile, out, 0666); err != nil {
+		return "", err
+	}
+	return dumpfile, err
+}

src/cmd/compile/internal/inline/inlheur/funcprops_test.go

@@ -234,18 +234,18 @@ func (dr *dumpReader) readObjBlob(delim string) (string, error) {
 // returns the resulting properties and function name. EOF is
 // signaled by a nil FuncProps return (with no error
 func (dr *dumpReader) readEntry() (fnInlHeur, encodedCallSiteTab, error) {
-	var fih fnInlHeur
+	var funcInlHeur fnInlHeur
 	var callsites encodedCallSiteTab
 	if !dr.scan() {
-		return fih, callsites, nil
+		return funcInlHeur, callsites, nil
 	}
 	// first line contains info about function: file/name/line
 	info := dr.curLine()
 	chunks := strings.Fields(info)
-	fih.file = chunks[0]
-	fih.fname = chunks[1]
-	if _, err := fmt.Sscanf(chunks[2], "%d", &fih.line); err != nil {
-		return fih, callsites, fmt.Errorf("scanning line %q: %v", info, err)
+	funcInlHeur.file = chunks[0]
+	funcInlHeur.fname = chunks[1]
+	if _, err := fmt.Sscanf(chunks[2], "%d", &funcInlHeur.line); err != nil {
+		return funcInlHeur, callsites, fmt.Errorf("scanning line %q: %v", info, err)
 	}
 	// consume comments until and including delimiter
 	for {
@@ -262,9 +262,9 @@ func (dr *dumpReader) readEntry() (fnInlHeur, encodedCallSiteTab, error) {
 	line := dr.curLine()
 	fp := &FuncProps{}
 	if err := json.Unmarshal([]byte(line), fp); err != nil {
-		return fih, callsites, err
+		return funcInlHeur, callsites, err
 	}
-	fih.props = fp
+	funcInlHeur.props = fp
 
 	// Consume callsites.
 	callsites = make(encodedCallSiteTab)
@@ -276,29 +276,29 @@ func (dr *dumpReader) readEntry() (fnInlHeur, encodedCallSiteTab, error) {
 		// expected format: "// callsite: <expanded pos> flagstr <desc> flagval <flags> score <score> mask <scoremask> maskstr <scoremaskstring>"
 		fields := strings.Fields(line)
 		if len(fields) != 12 {
-			return fih, nil, fmt.Errorf("malformed callsite (nf=%d) %s line %d: %s", len(fields), dr.p, dr.ln, line)
+			return funcInlHeur, nil, fmt.Errorf("malformed callsite (nf=%d) %s line %d: %s", len(fields), dr.p, dr.ln, line)
 		}
 		if fields[2] != "flagstr" || fields[4] != "flagval" || fields[6] != "score" || fields[8] != "mask" || fields[10] != "maskstr" {
-			return fih, nil, fmt.Errorf("malformed callsite %s line %d: %s",
+			return funcInlHeur, nil, fmt.Errorf("malformed callsite %s line %d: %s",
 				dr.p, dr.ln, line)
 		}
 		tag := fields[1]
 		flagstr := fields[5]
 		flags, err := strconv.Atoi(flagstr)
 		if err != nil {
-			return fih, nil, fmt.Errorf("bad flags val %s line %d: %q err=%v",
+			return funcInlHeur, nil, fmt.Errorf("bad flags val %s line %d: %q err=%v",
 				dr.p, dr.ln, line, err)
 		}
 		scorestr := fields[7]
 		score, err2 := strconv.Atoi(scorestr)
 		if err2 != nil {
-			return fih, nil, fmt.Errorf("bad score val %s line %d: %q err=%v",
+			return funcInlHeur, nil, fmt.Errorf("bad score val %s line %d: %q err=%v",
 				dr.p, dr.ln, line, err2)
 		}
 		maskstr := fields[9]
 		mask, err3 := strconv.Atoi(maskstr)
 		if err3 != nil {
-			return fih, nil, fmt.Errorf("bad mask val %s line %d: %q err=%v",
+			return funcInlHeur, nil, fmt.Errorf("bad mask val %s line %d: %q err=%v",
 				dr.p, dr.ln, line, err3)
 		}
 		callsites[tag] = propsAndScore{
@@ -312,10 +312,10 @@ func (dr *dumpReader) readEntry() (fnInlHeur, encodedCallSiteTab, error) {
 	dr.scan()
 	line = dr.curLine()
 	if line != fnDelimiter {
-		return fih, nil, fmt.Errorf("malformed testcase file %q, missing delimiter %q", dr.p, fnDelimiter)
+		return funcInlHeur, nil, fmt.Errorf("malformed testcase file %q, missing delimiter %q", dr.p, fnDelimiter)
 	}
 
-	return fih, callsites, nil
+	return funcInlHeur, callsites, nil
 }
 
 // gatherPropsDumpForFile builds the specified testcase 'testcase' from
@@ -346,6 +346,9 @@ func gatherPropsDumpForFile(t *testing.T, testcase string, td string) (string, e
 	run := []string{testenv.GoToolPath(t), "build",
 		"-gcflags=-d=dumpinlfuncprops=" + dumpfile, "-o", outpath, gopath}
 	out, err := testenv.Command(t, run[0], run[1:]...).CombinedOutput()
+	if err != nil {
+		t.Logf("compile command: %+v", run)
+	}
 	if strings.TrimSpace(string(out)) != "" {
 		t.Logf("%s", out)
 	}

src/cmd/compile/internal/inline/inlheur/names.go

@@ -0,0 +1,129 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package inlheur
+
+import (
+	"cmd/compile/internal/ir"
+	"go/constant"
+)
+
+// nameFinder provides a set of "isXXX" query methods for clients to
+// ask whether a given AST node corresponds to a function, a constant
+// value, and so on. These methods use an underlying ir.ReassignOracle
+// to return more precise results in cases where an "interesting"
+// value is assigned to a singly-defined local temp. Example:
+//
+//	const q = 101
+//	fq := func() int { return q }
+//	copyOfConstant := q
+//	copyOfFunc := f
+//	interestingCall(copyOfConstant, copyOfFunc)
+//
+// A name finder query method invoked on the arguments being passed to
+// "interestingCall" will be able detect that 'copyOfConstant' always
+// evaluates to a constant (even though it is in fact a PAUTO local
+// variable). A given nameFinder can also operate without using
+// ir.ReassignOracle (in cases where it is not practical to look
+// at the entire function); in such cases queries will still work
+// for explicit constant values and functions.
+type nameFinder struct {
+	ro *ir.ReassignOracle
+}
+
+// newNameFinder returns a new nameFinder object with a reassignment
+// oracle initialized based on the function fn, or if fn is nil,
+// without an underlying ReassignOracle.
+func newNameFinder(fn *ir.Func) *nameFinder {
+	var ro *ir.ReassignOracle
+	if fn != nil {
+		ro = &ir.ReassignOracle{}
+		ro.Init(fn)
+	}
+	return &nameFinder{ro: ro}
+}
+
+// funcName returns the *ir.Name for the func or method
+// corresponding to node 'n', or nil if n can't be proven
+// to contain a function value.
+func (nf *nameFinder) funcName(n ir.Node) *ir.Name {
+	sv := n
+	if nf.ro != nil {
+		sv = nf.ro.StaticValue(n)
+	}
+	if name := ir.StaticCalleeName(sv); name != nil {
+		return name
+	}
+	return nil
+}
+
+// isAllocatedMem returns true if node n corresponds to a memory
+// allocation expression (make, new, or equivalent).
+func (nf *nameFinder) isAllocatedMem(n ir.Node) bool {
+	sv := n
+	if nf.ro != nil {
+		sv = nf.ro.StaticValue(n)
+	}
+	switch sv.Op() {
+	case ir.OMAKESLICE, ir.ONEW, ir.OPTRLIT, ir.OSLICELIT:
+		return true
+	}
+	return false
+}
+
+// constValue returns the underlying constant.Value for an AST node n
+// if n is itself a constant value/expr, or if n is a singly assigned
+// local containing constant expr/value (or nil not constant).
+func (nf *nameFinder) constValue(n ir.Node) constant.Value {
+	sv := n
+	if nf.ro != nil {
+		sv = nf.ro.StaticValue(n)
+	}
+	if sv.Op() == ir.OLITERAL {
+		return sv.Val()
+	}
+	return nil
+}
+
+// isNil returns whether n is nil (or singly
+// assigned local containing nil).
+func (nf *nameFinder) isNil(n ir.Node) bool {
+	sv := n
+	if nf.ro != nil {
+		sv = nf.ro.StaticValue(n)
+	}
+	return sv.Op() == ir.ONIL
+}
+
+func (nf *nameFinder) staticValue(n ir.Node) ir.Node {
+	if nf.ro == nil {
+		return n
+	}
+	return nf.ro.StaticValue(n)
+}
+
+func (nf *nameFinder) reassigned(n *ir.Name) bool {
+	if nf.ro == nil {
+		return true
+	}
+	return nf.ro.Reassigned(n)
+}
+
+func (nf *nameFinder) isConcreteConvIface(n ir.Node) bool {
+	sv := n
+	if nf.ro != nil {
+		sv = nf.ro.StaticValue(n)
+	}
+	if sv.Op() != ir.OCONVIFACE {
+		return false
+	}
+	return !sv.(*ir.ConvExpr).X.Type().IsInterface()
+}
+
+func isSameFuncName(v1, v2 *ir.Name) bool {
+	// NB: there are a few corner cases where pointer equality
+	// doesn't work here, but this should be good enough for
+	// our purposes here.
+	return v1 == v2
+}

src/cmd/compile/internal/inline/inlheur/score_callresult_uses.go

@@ -46,10 +46,7 @@ type resultUseAnalyzer struct {
 // rescoreBasedOnCallResultUses examines how call results are used,
 // and tries to update the scores of calls based on how their results
 // are used in the function.
-func rescoreBasedOnCallResultUses(fn *ir.Func, resultNameTab map[*ir.Name]resultPropAndCS, cstab CallSiteTab) {
-	if os.Getenv("THANM_DEBUG") != "" {
-		return
-	}
+func (csa *callSiteAnalyzer) rescoreBasedOnCallResultUses(fn *ir.Func, resultNameTab map[*ir.Name]resultPropAndCS, cstab CallSiteTab) {
 	enableDebugTraceIfEnv()
 	rua := &resultUseAnalyzer{
 		resultNameTab:    resultNameTab,
@@ -68,7 +65,7 @@ func rescoreBasedOnCallResultUses(fn *ir.Func, resultNameTab map[*ir.Name]result
 	disableDebugTrace()
 }
 
-func examineCallResults(cs *CallSite, resultNameTab map[*ir.Name]resultPropAndCS) {
+func (csa *callSiteAnalyzer) examineCallResults(cs *CallSite, resultNameTab map[*ir.Name]resultPropAndCS) map[*ir.Name]resultPropAndCS {
 	if debugTrace&debugTraceScoring != 0 {
 		fmt.Fprintf(os.Stderr, "=-= examining call results for %q\n",
 			EncodeCallSiteKey(cs))
@@ -82,7 +79,7 @@ func examineCallResults(cs *CallSite, resultNameTab map[*ir.Name]resultPropAndCS
 	//
 	names, autoTemps, props := namesDefined(cs)
 	if len(names) == 0 {
-		return
+		return resultNameTab
 	}
 
 	if debugTrace&debugTraceScoring != 0 {
@@ -106,10 +103,12 @@ func examineCallResults(cs *CallSite, resultNameTab map[*ir.Name]resultPropAndCS
 		if rprop&interesting == 0 {
 			continue
 		}
-		if ir.Reassigned(n) {
+		if csa.nameFinder.reassigned(n) {
 			continue
 		}
-		if _, ok := resultNameTab[n]; ok {
+		if resultNameTab == nil {
+			resultNameTab = make(map[*ir.Name]resultPropAndCS)
+		} else if _, ok := resultNameTab[n]; ok {
 			panic("should never happen")
 		}
 		entry := resultPropAndCS{
@@ -124,6 +123,7 @@ func examineCallResults(cs *CallSite, resultNameTab map[*ir.Name]resultPropAndCS
 			fmt.Fprintf(os.Stderr, "=-= add resultNameTab table entry n=%v autotemp=%v props=%s\n", n, autoTemps[idx], rprop.String())
 		}
 	}
+	return resultNameTab
 }
 
 // namesDefined returns a list of ir.Name's corresponding to locals
@@ -150,17 +150,17 @@ func namesDefined(cs *CallSite) ([]*ir.Name, []*ir.Name, *FuncProps) {
 	if cs.Assign == nil {
 		return nil, nil, nil
 	}
-	fih, ok := fpmap[cs.Callee]
+	funcInlHeur, ok := fpmap[cs.Callee]
 	if !ok {
 		// TODO: add an assert/panic here.
 		return nil, nil, nil
 	}
-	if len(fih.props.ResultFlags) == 0 {
+	if len(funcInlHeur.props.ResultFlags) == 0 {
 		return nil, nil, nil
 	}
 
 	// Single return case.
-	if len(fih.props.ResultFlags) == 1 {
+	if len(funcInlHeur.props.ResultFlags) == 1 {
 		asgn, ok := cs.Assign.(*ir.AssignStmt)
 		if !ok {
 			return nil, nil, nil
@@ -170,7 +170,7 @@ func namesDefined(cs *CallSite) ([]*ir.Name, []*ir.Name, *FuncProps) {
 		if !ok {
 			return nil, nil, nil
 		}
-		return []*ir.Name{aname}, []*ir.Name{nil}, fih.props
+		return []*ir.Name{aname}, []*ir.Name{nil}, funcInlHeur.props
 	}
 
 	// Multi-return case
@@ -178,8 +178,8 @@ func namesDefined(cs *CallSite) ([]*ir.Name, []*ir.Name, *FuncProps) {
 	if !ok || !asgn.Def {
 		return nil, nil, nil
 	}
-	userVars := make([]*ir.Name, len(fih.props.ResultFlags))
-	autoTemps := make([]*ir.Name, len(fih.props.ResultFlags))
+	userVars := make([]*ir.Name, len(funcInlHeur.props.ResultFlags))
+	autoTemps := make([]*ir.Name, len(funcInlHeur.props.ResultFlags))
 	for idx, x := range asgn.Lhs {
 		if n, ok := x.(*ir.Name); ok {
 			userVars[idx] = n
@@ -198,7 +198,7 @@ func namesDefined(cs *CallSite) ([]*ir.Name, []*ir.Name, *FuncProps) {
 			return nil, nil, nil
 		}
 	}
-	return userVars, autoTemps, fih.props
+	return userVars, autoTemps, funcInlHeur.props
 }
 
 func (rua *resultUseAnalyzer) nodeVisitPost(n ir.Node) {
@@ -267,10 +267,8 @@ func (rua *resultUseAnalyzer) callTargetCheckResults(call ir.Node) {
 				rua.fn.Sym().Name, rname)
 		}
 		if cs := rua.returnHasProp(rname, ResultIsConcreteTypeConvertedToInterface); cs != nil {
-			// FIXME: add cond level support here
-			adj := passConcreteToItfCallAdj
-			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
-			adj = callResultRescoreAdj
+
+			adj := returnFeedsConcreteToInterfaceCallAdj
 			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
 		}
 	case ir.OCALLFUNC:
@@ -285,17 +283,12 @@ func (rua *resultUseAnalyzer) callTargetCheckResults(call ir.Node) {
 			}
 		}
 		if cs := rua.returnHasProp(rname, ResultAlwaysSameInlinableFunc); cs != nil {
-			// FIXME: add cond level support here
-			adj := passInlinableFuncToIndCallAdj
-			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
-			adj = callResultRescoreAdj
+			adj := returnFeedsInlinableFuncToIndCallAdj
 			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
 		} else if cs := rua.returnHasProp(rname, ResultAlwaysSameFunc); cs != nil {
-			// FIXME: add cond level support here
-			adj := passFuncToIndCallAdj
-			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
-			adj = callResultRescoreAdj
+			adj := returnFeedsFuncToIndCallAdj
 			cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
+
 		}
 	}
 }
@@ -351,10 +344,7 @@ func (rua *resultUseAnalyzer) foldCheckResults(cond ir.Node) {
 	if !ShouldFoldIfNameConstant(cond, namesUsed) {
 		return
 	}
-	// FIXME: add cond level support here
-	adj := passConstToIfAdj
-	cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
-	adj = callResultRescoreAdj
+	adj := returnFeedsConstToIfAdj
 	cs.Score, cs.ScoreMask = adjustScore(adj, cs.Score, cs.ScoreMask)
 }
 

src/cmd/compile/internal/inline/inlheur/scoreadjusttyp_string.go

@@ -20,29 +20,33 @@ func _() {
 	_ = x[passFuncToNestedIndCallAdj-256]
 	_ = x[passInlinableFuncToIndCallAdj-512]
 	_ = x[passInlinableFuncToNestedIndCallAdj-1024]
-	_ = x[callResultRescoreAdj-2048]
-	_ = x[lastAdj-2048]
+	_ = x[returnFeedsConstToIfAdj-2048]
+	_ = x[returnFeedsFuncToIndCallAdj-4096]
+	_ = x[returnFeedsInlinableFuncToIndCallAdj-8192]
+	_ = x[returnFeedsConcreteToInterfaceCallAdj-16384]
 }
 
 var _scoreAdjustTyp_value = [...]uint64{
-	0x1,   /* panicPathAdj */
-	0x2,   /* initFuncAdj */
-	0x4,   /* inLoopAdj */
-	0x8,   /* passConstToIfAdj */
-	0x10,  /* passConstToNestedIfAdj */
-	0x20,  /* passConcreteToItfCallAdj */
-	0x40,  /* passConcreteToNestedItfCallAdj */
-	0x80,  /* passFuncToIndCallAdj */
-	0x100, /* passFuncToNestedIndCallAdj */
-	0x200, /* passInlinableFuncToIndCallAdj */
-	0x400, /* passInlinableFuncToNestedIndCallAdj */
-	0x800, /* callResultRescoreAdj */
-	0x800, /* lastAdj */
+	0x1,    /* panicPathAdj */
+	0x2,    /* initFuncAdj */
+	0x4,    /* inLoopAdj */
+	0x8,    /* passConstToIfAdj */
+	0x10,   /* passConstToNestedIfAdj */
+	0x20,   /* passConcreteToItfCallAdj */
+	0x40,   /* passConcreteToNestedItfCallAdj */
+	0x80,   /* passFuncToIndCallAdj */
+	0x100,  /* passFuncToNestedIndCallAdj */
+	0x200,  /* passInlinableFuncToIndCallAdj */
+	0x400,  /* passInlinableFuncToNestedIndCallAdj */
+	0x800,  /* returnFeedsConstToIfAdj */
+	0x1000, /* returnFeedsFuncToIndCallAdj */
+	0x2000, /* returnFeedsInlinableFuncToIndCallAdj */
+	0x4000, /* returnFeedsConcreteToInterfaceCallAdj */
 }
 
-const _scoreAdjustTyp_name = "panicPathAdjinitFuncAdjinLoopAdjpassConstToIfAdjpassConstToNestedIfAdjpassConcreteToItfCallAdjpassConcreteToNestedItfCallAdjpassFuncToIndCallAdjpassFuncToNestedIndCallAdjpassInlinableFuncToIndCallAdjpassInlinableFuncToNestedIndCallAdjcallResultRescoreAdjlastAdj"
+const _scoreAdjustTyp_name = "panicPathAdjinitFuncAdjinLoopAdjpassConstToIfAdjpassConstToNestedIfAdjpassConcreteToItfCallAdjpassConcreteToNestedItfCallAdjpassFuncToIndCallAdjpassFuncToNestedIndCallAdjpassInlinableFuncToIndCallAdjpassInlinableFuncToNestedIndCallAdjreturnFeedsConstToIfAdjreturnFeedsFuncToIndCallAdjreturnFeedsInlinableFuncToIndCallAdjreturnFeedsConcreteToInterfaceCallAdj"
 
-var _scoreAdjustTyp_index = [...]uint16{0, 12, 23, 32, 48, 70, 94, 124, 144, 170, 199, 234, 254, 261}
+var _scoreAdjustTyp_index = [...]uint16{0, 12, 23, 32, 48, 70, 94, 124, 144, 170, 199, 234, 257, 284, 320, 357}
 
 func (i scoreAdjustTyp) String() string {
 	var b bytes.Buffer

src/cmd/compile/internal/inline/inlheur/scoring.go

@@ -8,21 +8,47 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/pgo"
-	"cmd/compile/internal/typecheck"
 	"cmd/compile/internal/types"
 	"fmt"
 	"os"
 	"sort"
+	"strconv"
+	"strings"
 )
 
 // These constants enumerate the set of possible ways/scenarios
 // in which we'll adjust the score of a given callsite.
 type scoreAdjustTyp uint
 
+// These constants capture the various ways in which the inliner's
+// scoring phase can adjust a callsite score based on heuristics. They
+// fall broadly into three categories:
+//
+// 1) adjustments based solely on the callsite context (ex: call
+// appears on panic path)
+//
+// 2) adjustments that take into account specific interesting values
+// passed at a call site (ex: passing a constant that could result in
+// cprop/deadcode in the caller)
+//
+// 3) adjustments that take into account values returned from the call
+// at a callsite (ex: call always returns the same inlinable function,
+// and return value flows unmodified into an indirect call)
+//
+// For categories 2 and 3 above, each adjustment can have either a
+// "must" version and a "may" version (but not both). Here the idea is
+// that in the "must" version the value flow is unconditional: if the
+// callsite executes, then the condition we're interested in (ex:
+// param feeding call) is guaranteed to happen. For the "may" version,
+// there may be control flow that could cause the benefit to be
+// bypassed.
 const (
+	// Category 1 adjustments (see above)
 	panicPathAdj scoreAdjustTyp = (1 << iota)
 	initFuncAdj
 	inLoopAdj
+
+	// Category 2 adjustments (see above).
 	passConstToIfAdj
 	passConstToNestedIfAdj
 	passConcreteToItfCallAdj
@@ -31,8 +57,14 @@ const (
 	passFuncToNestedIndCallAdj
 	passInlinableFuncToIndCallAdj
 	passInlinableFuncToNestedIndCallAdj
-	callResultRescoreAdj
-	lastAdj scoreAdjustTyp = callResultRescoreAdj
+
+	// Category 3 adjustments.
+	returnFeedsConstToIfAdj
+	returnFeedsFuncToIndCallAdj
+	returnFeedsInlinableFuncToIndCallAdj
+	returnFeedsConcreteToInterfaceCallAdj
+
+	sentinelScoreAdj // sentinel; not a real adjustment
 )
 
 // This table records the specific values we use to adjust call
@@ -42,18 +74,71 @@ const (
 // what value for each one produces the best performance.
 
 var adjValues = map[scoreAdjustTyp]int{
-	panicPathAdj:                        40,
-	initFuncAdj:                         20,
-	inLoopAdj:                           -5,
-	passConstToIfAdj:                    -20,
-	passConstToNestedIfAdj:              -15,
-	passConcreteToItfCallAdj:            -30,
-	passConcreteToNestedItfCallAdj:      -25,
-	passFuncToIndCallAdj:                -25,
-	passFuncToNestedIndCallAdj:          -20,
-	passInlinableFuncToIndCallAdj:       -45,
-	passInlinableFuncToNestedIndCallAdj: -40,
-	callResultRescoreAdj:                0,
+	panicPathAdj:                          40,
+	initFuncAdj:                           20,
+	inLoopAdj:                             -5,
+	passConstToIfAdj:                      -20,
+	passConstToNestedIfAdj:                -15,
+	passConcreteToItfCallAdj:              -30,
+	passConcreteToNestedItfCallAdj:        -25,
+	passFuncToIndCallAdj:                  -25,
+	passFuncToNestedIndCallAdj:            -20,
+	passInlinableFuncToIndCallAdj:         -45,
+	passInlinableFuncToNestedIndCallAdj:   -40,
+	returnFeedsConstToIfAdj:               -15,
+	returnFeedsFuncToIndCallAdj:           -25,
+	returnFeedsInlinableFuncToIndCallAdj:  -40,
+	returnFeedsConcreteToInterfaceCallAdj: -25,
+}
+
+// SetupScoreAdjustments interprets the value of the -d=inlscoreadj
+// debugging option, if set. The value of this flag is expected to be
+// a series of "/"-separated clauses of the form adj1:value1. Example:
+// -d=inlscoreadj=inLoopAdj=0/passConstToIfAdj=-99
+func SetupScoreAdjustments() {
+	if base.Debug.InlScoreAdj == "" {
+		return
+	}
+	if err := parseScoreAdj(base.Debug.InlScoreAdj); err != nil {
+		base.Fatalf("malformed -d=inlscoreadj argument %q: %v",
+			base.Debug.InlScoreAdj, err)
+	}
+}
+
+func adjStringToVal(s string) (scoreAdjustTyp, bool) {
+	for adj := scoreAdjustTyp(1); adj < sentinelScoreAdj; adj <<= 1 {
+		if adj.String() == s {
+			return adj, true
+		}
+	}
+	return 0, false
+}
+
+func parseScoreAdj(val string) error {
+	clauses := strings.Split(val, "/")
+	if len(clauses) == 0 {
+		return fmt.Errorf("no clauses")
+	}
+	for _, clause := range clauses {
+		elems := strings.Split(clause, ":")
+		if len(elems) < 2 {
+			return fmt.Errorf("clause %q: expected colon", clause)
+		}
+		if len(elems) != 2 {
+			return fmt.Errorf("clause %q has %d elements, wanted 2", clause,
+				len(elems))
+		}
+		adj, ok := adjStringToVal(elems[0])
+		if !ok {
+			return fmt.Errorf("clause %q: unknown adjustment", clause)
+		}
+		val, err := strconv.Atoi(elems[1])
+		if err != nil {
+			return fmt.Errorf("clause %q: malformed value: %v", clause, err)
+		}
+		adjValues[adj] = val
+	}
+	return nil
 }
 
 func adjValue(x scoreAdjustTyp) int {
@@ -64,7 +149,7 @@ func adjValue(x scoreAdjustTyp) int {
 	}
 }
 
-var mayMust = [...]struct{ may, must scoreAdjustTyp }{
+var mayMustAdj = [...]struct{ may, must scoreAdjustTyp }{
 	{may: passConstToNestedIfAdj, must: passConstToIfAdj},
 	{may: passConcreteToNestedItfCallAdj, must: passConcreteToItfCallAdj},
 	{may: passFuncToNestedIndCallAdj, must: passFuncToNestedIndCallAdj},
@@ -80,7 +165,7 @@ func isMust(x scoreAdjustTyp) bool {
 }
 
 func mayToMust(x scoreAdjustTyp) scoreAdjustTyp {
-	for _, v := range mayMust {
+	for _, v := range mayMustAdj {
 		if x == v.may {
 			return v.must
 		}
@@ -89,7 +174,7 @@ func mayToMust(x scoreAdjustTyp) scoreAdjustTyp {
 }
 
 func mustToMay(x scoreAdjustTyp) scoreAdjustTyp {
-	for _, v := range mayMust {
+	for _, v := range mayMustAdj {
 		if x == v.must {
 			return v.may
 		}
@@ -97,12 +182,18 @@ func mustToMay(x scoreAdjustTyp) scoreAdjustTyp {
 	return 0
 }
 
-// computeCallSiteScore takes a given call site whose ir node is 'call' and
-// callee function is 'callee' and with previously computed call site
-// properties 'csflags', then computes a score for the callsite that
-// combines the size cost of the callee with heuristics based on
-// previously parameter and function properties.
-func computeCallSiteScore(callee *ir.Func, calleeProps *FuncProps, call ir.Node, csflags CSPropBits) (int, scoreAdjustTyp) {
+// computeCallSiteScore takes a given call site whose ir node is
+// 'call' and callee function is 'callee' and with previously computed
+// call site properties 'csflags', then computes a score for the
+// callsite that combines the size cost of the callee with heuristics
+// based on previously computed argument and function properties,
+// then stores the score and the adjustment mask in the appropriate
+// fields in 'cs'
+func (cs *CallSite) computeCallSiteScore(csa *callSiteAnalyzer, calleeProps *FuncProps) {
+	callee := cs.Callee
+	csflags := cs.Flags
+	call := cs.Call
+
 	// Start with the size-based score for the callee.
 	score := int(callee.Inl.Cost)
 	var tmask scoreAdjustTyp
@@ -125,29 +216,38 @@ func computeCallSiteScore(callee *ir.Func, calleeProps *FuncProps, call ir.Node,
 		score, tmask = adjustScore(inLoopAdj, score, tmask)
 	}
 
+	// Stop here if no callee props.
+	if calleeProps == nil {
+		cs.Score, cs.ScoreMask = score, tmask
+		return
+	}
+
 	// Walk through the actual expressions being passed at the call.
 	calleeRecvrParms := callee.Type().RecvParams()
-	ce := call.(*ir.CallExpr)
-	for idx := range ce.Args {
+	for idx := range call.Args {
 		// ignore blanks
 		if calleeRecvrParms[idx].Sym == nil ||
 			calleeRecvrParms[idx].Sym.IsBlank() {
 			continue
 		}
-		arg := ce.Args[idx]
+		arg := call.Args[idx]
 		pflag := calleeProps.ParamFlags[idx]
 		if debugTrace&debugTraceScoring != 0 {
 			fmt.Fprintf(os.Stderr, "=-= arg %d of %d: val %v flags=%s\n",
-				idx, len(ce.Args), arg, pflag.String())
-		}
-		_, islit := isLiteral(arg)
-		iscci := isConcreteConvIface(arg)
-		fname, isfunc, _ := isFuncName(arg)
-		if debugTrace&debugTraceScoring != 0 {
-			fmt.Fprintf(os.Stderr, "=-= isLit=%v iscci=%v isfunc=%v for arg %v\n", islit, iscci, isfunc, arg)
+				idx, len(call.Args), arg, pflag.String())
 		}
 
-		if islit {
+		if len(cs.ArgProps) == 0 {
+			continue
+		}
+		argProps := cs.ArgProps[idx]
+
+		if debugTrace&debugTraceScoring != 0 {
+			fmt.Fprintf(os.Stderr, "=-= arg %d props %s value %v\n",
+				idx, argProps.String(), arg)
+		}
+
+		if argProps&ActualExprConstant != 0 {
 			if pflag&ParamMayFeedIfOrSwitch != 0 {
 				score, tmask = adjustScore(passConstToNestedIfAdj, score, tmask)
 			}
@@ -156,7 +256,7 @@ func computeCallSiteScore(callee *ir.Func, calleeProps *FuncProps, call ir.Node,
 			}
 		}
 
-		if iscci {
+		if argProps&ActualExprIsConcreteConvIface != 0 {
 			// FIXME: ideally here it would be nice to make a
 			// distinction between the inlinable case and the
 			// non-inlinable case, but this is hard to do. Example:
@@ -189,10 +289,10 @@ func computeCallSiteScore(callee *ir.Func, calleeProps *FuncProps, call ir.Node,
 			}
 		}
 
-		if isfunc {
+		if argProps&(ActualExprIsFunc|ActualExprIsInlinableFunc) != 0 {
 			mayadj := passFuncToNestedIndCallAdj
 			mustadj := passFuncToIndCallAdj
-			if fn := fname.Func; fn != nil && typecheck.HaveInlineBody(fn) {
+			if argProps&ActualExprIsInlinableFunc != 0 {
 				mayadj = passInlinableFuncToNestedIndCallAdj
 				mustadj = passInlinableFuncToIndCallAdj
 			}
@@ -205,7 +305,7 @@ func computeCallSiteScore(callee *ir.Func, calleeProps *FuncProps, call ir.Node,
 		}
 	}
 
-	return score, tmask
+	cs.Score, cs.ScoreMask = score, tmask
 }
 
 func adjustScore(typ scoreAdjustTyp, score int, mask scoreAdjustTyp) (int, scoreAdjustTyp) {
@@ -237,6 +337,280 @@ func adjustScore(typ scoreAdjustTyp, score int, mask scoreAdjustTyp) (int, score
 	return score, mask
 }
 
+var resultFlagToPositiveAdj map[ResultPropBits]scoreAdjustTyp
+var paramFlagToPositiveAdj map[ParamPropBits]scoreAdjustTyp
+
+func setupFlagToAdjMaps() {
+	resultFlagToPositiveAdj = map[ResultPropBits]scoreAdjustTyp{
+		ResultIsAllocatedMem:     returnFeedsConcreteToInterfaceCallAdj,
+		ResultAlwaysSameFunc:     returnFeedsFuncToIndCallAdj,
+		ResultAlwaysSameConstant: returnFeedsConstToIfAdj,
+	}
+	paramFlagToPositiveAdj = map[ParamPropBits]scoreAdjustTyp{
+		ParamMayFeedInterfaceMethodCall: passConcreteToNestedItfCallAdj,
+		ParamFeedsInterfaceMethodCall:   passConcreteToItfCallAdj,
+		ParamMayFeedIndirectCall:        passInlinableFuncToNestedIndCallAdj,
+		ParamFeedsIndirectCall:          passInlinableFuncToIndCallAdj,
+	}
+}
+
+// LargestNegativeScoreAdjustment tries to estimate the largest possible
+// negative score adjustment that could be applied to a call of the
+// function with the specified props. Example:
+//
+//	func foo() {                  func bar(x int, p *int) int {
+//	   ...                          if x < 0 { *p = x }
+//	}                               return 99
+//	                              }
+//
+// Function 'foo' above on the left has no interesting properties,
+// thus as a result the most we'll adjust any call to is the value for
+// "call in loop". If the calculated cost of the function is 150, and
+// the in-loop adjustment is 5 (for example), then there is not much
+// point treating it as inlinable. On the other hand "bar" has a param
+// property (parameter "x" feeds unmodified to an "if" statement") and
+// a return property (always returns same constant) meaning that a
+// given call _could_ be rescored down as much as -35 points-- thus if
+// the size of "bar" is 100 (for example) then there is at least a
+// chance that scoring will enable inlining.
+func LargestNegativeScoreAdjustment(fn *ir.Func, props *FuncProps) int {
+	if resultFlagToPositiveAdj == nil {
+		setupFlagToAdjMaps()
+	}
+	var tmask scoreAdjustTyp
+	score := adjValues[inLoopAdj] // any call can be in a loop
+	for _, pf := range props.ParamFlags {
+		if adj, ok := paramFlagToPositiveAdj[pf]; ok {
+			score, tmask = adjustScore(adj, score, tmask)
+		}
+	}
+	for _, rf := range props.ResultFlags {
+		if adj, ok := resultFlagToPositiveAdj[rf]; ok {
+			score, tmask = adjustScore(adj, score, tmask)
+		}
+	}
+
+	if debugTrace&debugTraceScoring != 0 {
+		fmt.Fprintf(os.Stderr, "=-= largestScore(%v) is %d\n",
+			fn, score)
+	}
+
+	return score
+}
+
+// LargestPositiveScoreAdjustment tries to estimate the largest possible
+// positive score adjustment that could be applied to a given callsite.
+// At the moment we don't have very many positive score adjustments, so
+// this is just hard-coded, not table-driven.
+func LargestPositiveScoreAdjustment(fn *ir.Func) int {
+	return adjValues[panicPathAdj] + adjValues[initFuncAdj]
+}
+
+// callSiteTab contains entries for each call in the function
+// currently being processed by InlineCalls; this variable will either
+// be set to 'cstabCache' below (for non-inlinable routines) or to the
+// local 'cstab' entry in the fnInlHeur object for inlinable routines.
+//
+// NOTE: this assumes that inlining operations are happening in a serial,
+// single-threaded fashion,f which is true today but probably won't hold
+// in the future (for example, we might want to score the callsites
+// in multiple functions in parallel); if the inliner evolves in this
+// direction we'll need to come up with a different approach here.
+var callSiteTab CallSiteTab
+
+// scoreCallsCache caches a call site table and call site list between
+// invocations of ScoreCalls so that we can reuse previously allocated
+// storage.
+var scoreCallsCache scoreCallsCacheType
+
+type scoreCallsCacheType struct {
+	tab CallSiteTab
+	csl []*CallSite
+}
+
+// ScoreCalls assigns numeric scores to each of the callsites in
+// function 'fn'; the lower the score, the more helpful we think it
+// will be to inline.
+//
+// Unlike a lot of the other inline heuristics machinery, callsite
+// scoring can't be done as part of the CanInline call for a function,
+// due to fact that we may be working on a non-trivial SCC. So for
+// example with this SCC:
+//
+//	func foo(x int) {           func bar(x int, f func()) {
+//	  if x != 0 {                  f()
+//	    bar(x, func(){})           foo(x-1)
+//	  }                         }
+//	}
+//
+// We don't want to perform scoring for the 'foo' call in "bar" until
+// after foo has been analyzed, but it's conceivable that CanInline
+// might visit bar before foo for this SCC.
+func ScoreCalls(fn *ir.Func) {
+	if len(fn.Body) == 0 {
+		return
+	}
+	enableDebugTraceIfEnv()
+
+	nameFinder := newNameFinder(fn)
+
+	if debugTrace&debugTraceScoring != 0 {
+		fmt.Fprintf(os.Stderr, "=-= ScoreCalls(%v)\n", ir.FuncName(fn))
+	}
+
+	// If this is an inlinable function, use the precomputed
+	// call site table for it. If the function wasn't an inline
+	// candidate, collect a callsite table for it now.
+	var cstab CallSiteTab
+	if funcInlHeur, ok := fpmap[fn]; ok {
+		cstab = funcInlHeur.cstab
+	} else {
+		if len(scoreCallsCache.tab) != 0 {
+			panic("missing call to ScoreCallsCleanup")
+		}
+		if scoreCallsCache.tab == nil {
+			scoreCallsCache.tab = make(CallSiteTab)
+		}
+		if debugTrace&debugTraceScoring != 0 {
+			fmt.Fprintf(os.Stderr, "=-= building cstab for non-inl func %s\n",
+				ir.FuncName(fn))
+		}
+		cstab = computeCallSiteTable(fn, fn.Body, scoreCallsCache.tab, nil, 0,
+			nameFinder)
+	}
+
+	csa := makeCallSiteAnalyzer(fn)
+	const doCallResults = true
+	csa.scoreCallsRegion(fn, fn.Body, cstab, doCallResults, nil)
+
+	disableDebugTrace()
+}
+
+// scoreCallsRegion assigns numeric scores to each of the callsites in
+// region 'region' within function 'fn'. This can be called on
+// an entire function, or with 'region' set to a chunk of
+// code corresponding to an inlined call.
+func (csa *callSiteAnalyzer) scoreCallsRegion(fn *ir.Func, region ir.Nodes, cstab CallSiteTab, doCallResults bool, ic *ir.InlinedCallExpr) {
+	if debugTrace&debugTraceScoring != 0 {
+		fmt.Fprintf(os.Stderr, "=-= scoreCallsRegion(%v, %s) len(cstab)=%d\n",
+			ir.FuncName(fn), region[0].Op().String(), len(cstab))
+	}
+
+	// Sort callsites to avoid any surprises with non deterministic
+	// map iteration order (this is probably not needed, but here just
+	// in case).
+	csl := scoreCallsCache.csl[:0]
+	for _, cs := range cstab {
+		csl = append(csl, cs)
+	}
+	scoreCallsCache.csl = csl[:0]
+	sort.Slice(csl, func(i, j int) bool {
+		return csl[i].ID < csl[j].ID
+	})
+
+	// Score each call site.
+	var resultNameTab map[*ir.Name]resultPropAndCS
+	for _, cs := range csl {
+		var cprops *FuncProps
+		fihcprops := false
+		desercprops := false
+		if funcInlHeur, ok := fpmap[cs.Callee]; ok {
+			cprops = funcInlHeur.props
+			fihcprops = true
+		} else if cs.Callee.Inl != nil {
+			cprops = DeserializeFromString(cs.Callee.Inl.Properties)
+			desercprops = true
+		} else {
+			if base.Debug.DumpInlFuncProps != "" {
+				fmt.Fprintf(os.Stderr, "=-= *** unable to score call to %s from %s\n", cs.Callee.Sym().Name, fmtFullPos(cs.Call.Pos()))
+				panic("should never happen")
+			} else {
+				continue
+			}
+		}
+		cs.computeCallSiteScore(csa, cprops)
+
+		if doCallResults {
+			if debugTrace&debugTraceScoring != 0 {
+				fmt.Fprintf(os.Stderr, "=-= examineCallResults at %s: flags=%d score=%d funcInlHeur=%v deser=%v\n", fmtFullPos(cs.Call.Pos()), cs.Flags, cs.Score, fihcprops, desercprops)
+			}
+			resultNameTab = csa.examineCallResults(cs, resultNameTab)
+		}
+
+		if debugTrace&debugTraceScoring != 0 {
+			fmt.Fprintf(os.Stderr, "=-= scoring call at %s: flags=%d score=%d funcInlHeur=%v deser=%v\n", fmtFullPos(cs.Call.Pos()), cs.Flags, cs.Score, fihcprops, desercprops)
+		}
+	}
+
+	if resultNameTab != nil {
+		csa.rescoreBasedOnCallResultUses(fn, resultNameTab, cstab)
+	}
+
+	disableDebugTrace()
+
+	if ic != nil && callSiteTab != nil {
+		// Integrate the calls from this cstab into the table for the caller.
+		if err := callSiteTab.merge(cstab); err != nil {
+			base.FatalfAt(ic.Pos(), "%v", err)
+		}
+	} else {
+		callSiteTab = cstab
+	}
+}
+
+// ScoreCallsCleanup resets the state of the callsite cache
+// once ScoreCalls is done with a function.
+func ScoreCallsCleanup() {
+	if base.Debug.DumpInlCallSiteScores != 0 {
+		if allCallSites == nil {
+			allCallSites = make(CallSiteTab)
+		}
+		for call, cs := range callSiteTab {
+			allCallSites[call] = cs
+		}
+	}
+	for k := range scoreCallsCache.tab {
+		delete(scoreCallsCache.tab, k)
+	}
+}
+
+// GetCallSiteScore returns the previously calculated score for call
+// within fn.
+func GetCallSiteScore(fn *ir.Func, call *ir.CallExpr) (int, bool) {
+	if funcInlHeur, ok := fpmap[fn]; ok {
+		if cs, ok := funcInlHeur.cstab[call]; ok {
+			return cs.Score, true
+		}
+	}
+	if cs, ok := callSiteTab[call]; ok {
+		return cs.Score, true
+	}
+	return 0, false
+}
+
+// BudgetExpansion returns the amount to relax/expand the base
+// inlining budget when the new inliner is turned on; the inliner
+// will add the returned value to the hairyness budget.
+//
+// Background: with the new inliner, the score for a given callsite
+// can be adjusted down by some amount due to heuristics, however we
+// won't know whether this is going to happen until much later after
+// the CanInline call. This function returns the amount to relax the
+// budget initially (to allow for a large score adjustment); later on
+// in RevisitInlinability we'll look at each individual function to
+// demote it if needed.
+func BudgetExpansion(maxBudget int32) int32 {
+	if base.Debug.InlBudgetSlack != 0 {
+		return int32(base.Debug.InlBudgetSlack)
+	}
+	// In the default case, return maxBudget, which will effectively
+	// double the budget from 80 to 160; this should be good enough
+	// for most cases.
+	return maxBudget
+}
+
+var allCallSites CallSiteTab
+
 // DumpInlCallSiteScores is invoked by the inliner if the debug flag
 // "-d=dumpinlcallsitescores" is set; it dumps out a human-readable
 // summary of all (potentially) inlinable callsites in the package,
@@ -249,7 +623,7 @@ func adjustScore(typ scoreAdjustTyp, score int, mask scoreAdjustTyp) (int, score
 //
 // Score  Adjustment  Status  Callee  CallerPos ScoreFlags
 // 115    40          DEMOTED cmd/compile/internal/abi.(*ABIParamAssignment).Offset     expand_calls.go:1679:14|6       panicPathAdj
-// 76     -5n           PROMOTED runtime.persistentalloc   mcheckmark.go:48:45|3   inLoopAdj
+// 76     -5n         PROMOTED runtime.persistentalloc   mcheckmark.go:48:45|3   inLoopAdj
 // 201    0           --- PGO  unicode.DecodeRuneInString        utf8.go:312:30|1
 // 7      -5          --- PGO  internal/abi.Name.DataChecked     type.go:625:22|0        inLoopAdj
 //
@@ -266,19 +640,31 @@ func adjustScore(typ scoreAdjustTyp, score int, mask scoreAdjustTyp) (int, score
 // we used to make adjustments to callsite score via heuristics.
 func DumpInlCallSiteScores(profile *pgo.Profile, budgetCallback func(fn *ir.Func, profile *pgo.Profile) (int32, bool)) {
 
-	fmt.Fprintf(os.Stdout, "# scores for package %s\n", types.LocalPkg.Path)
-	cstab := CallSiteTable()
-
-	genstatus := func(cs *CallSite, prof *pgo.Profile) string {
+	var indirectlyDueToPromotion func(cs *CallSite) bool
+	indirectlyDueToPromotion = func(cs *CallSite) bool {
+		bud, _ := budgetCallback(cs.Callee, profile)
 		hairyval := cs.Callee.Inl.Cost
-		bud, isPGO := budgetCallback(cs.Callee, prof)
+		score := int32(cs.Score)
+		if hairyval > bud && score <= bud {
+			return true
+		}
+		if cs.parent != nil {
+			return indirectlyDueToPromotion(cs.parent)
+		}
+		return false
+	}
+
+	genstatus := func(cs *CallSite) string {
+		hairyval := cs.Callee.Inl.Cost
+		bud, isPGO := budgetCallback(cs.Callee, profile)
 		score := int32(cs.Score)
 		st := "---"
-
+		expinl := false
 		switch {
 		case hairyval <= bud && score <= bud:
 			// "Normal" inlined case: hairy val sufficiently low that
 			// it would have been inlined anyway without heuristics.
+			expinl = true
 		case hairyval > bud && score > bud:
 			// "Normal" not inlined case: hairy val sufficiently high
 			// and scoring didn't lower it.
@@ -286,21 +672,35 @@ func DumpInlCallSiteScores(profile *pgo.Profile, budgetCallback func(fn *ir.Func
 			// Promoted: we would not have inlined it before, but
 			// after score adjustment we decided to inline.
 			st = "PROMOTED"
+			expinl = true
 		case hairyval <= bud && score > bud:
 			// Demoted: we would have inlined it before, but after
 			// score adjustment we decided not to inline.
 			st = "DEMOTED"
 		}
+		inlined := cs.aux&csAuxInlined != 0
+		indprom := false
+		if cs.parent != nil {
+			indprom = indirectlyDueToPromotion(cs.parent)
+		}
+		if inlined && indprom {
+			st += "|INDPROM"
+		}
+		if inlined && !expinl {
+			st += "|[NI?]"
+		} else if !inlined && expinl {
+			st += "|[IN?]"
+		}
 		if isPGO {
-			st += " PGO"
+			st += "|PGO"
 		}
 		return st
 	}
 
 	if base.Debug.DumpInlCallSiteScores != 0 {
-		sl := make([]*CallSite, 0, len(cstab))
-		for _, v := range cstab {
-			sl = append(sl, v)
+		var sl []*CallSite
+		for _, cs := range allCallSites {
+			sl = append(sl, cs)
 		}
 		sort.Slice(sl, func(i, j int) bool {
 			if sl[i].Score != sl[j].Score {
@@ -334,15 +734,17 @@ func DumpInlCallSiteScores(profile *pgo.Profile, budgetCallback func(fn *ir.Func
 		}
 
 		if len(sl) != 0 {
-			fmt.Fprintf(os.Stdout, "Score  Adjustment  Status  Callee  CallerPos Flags ScoreFlags\n")
+			fmt.Fprintf(os.Stdout, "# scores for package %s\n", types.LocalPkg.Path)
+			fmt.Fprintf(os.Stdout, "# Score  Adjustment  Status  Callee  CallerPos Flags ScoreFlags\n")
 		}
 		for _, cs := range sl {
 			hairyval := cs.Callee.Inl.Cost
 			adj := int32(cs.Score) - hairyval
+			nm := mkname(cs.Callee)
+			ecc := EncodeCallSiteKey(cs)
 			fmt.Fprintf(os.Stdout, "%d  %d\t%s\t%s\t%s\t%s\n",
-				cs.Score, adj, genstatus(cs, profile),
-				mkname(cs.Callee),
-				EncodeCallSiteKey(cs),
+				cs.Score, adj, genstatus(cs),
+				nm, ecc,
 				cs.ScoreMask.String())
 		}
 	}

src/cmd/compile/internal/inline/inlheur/serialize.go

@@ -6,18 +6,18 @@ package inlheur
 
 import "strings"
 
-func (fp *FuncProps) SerializeToString() string {
-	if fp == nil {
+func (funcProps *FuncProps) SerializeToString() string {
+	if funcProps == nil {
 		return ""
 	}
 	var sb strings.Builder
-	writeUleb128(&sb, uint64(fp.Flags))
-	writeUleb128(&sb, uint64(len(fp.ParamFlags)))
-	for _, pf := range fp.ParamFlags {
+	writeUleb128(&sb, uint64(funcProps.Flags))
+	writeUleb128(&sb, uint64(len(funcProps.ParamFlags)))
+	for _, pf := range funcProps.ParamFlags {
 		writeUleb128(&sb, uint64(pf))
 	}
-	writeUleb128(&sb, uint64(len(fp.ResultFlags)))
-	for _, rf := range fp.ResultFlags {
+	writeUleb128(&sb, uint64(len(funcProps.ResultFlags)))
+	for _, rf := range funcProps.ResultFlags {
 		writeUleb128(&sb, uint64(rf))
 	}
 	return sb.String()
@@ -27,24 +27,24 @@ func DeserializeFromString(s string) *FuncProps {
 	if len(s) == 0 {
 		return nil
 	}
-	var fp FuncProps
+	var funcProps FuncProps
 	var v uint64
 	sl := []byte(s)
 	v, sl = readULEB128(sl)
-	fp.Flags = FuncPropBits(v)
+	funcProps.Flags = FuncPropBits(v)
 	v, sl = readULEB128(sl)
-	fp.ParamFlags = make([]ParamPropBits, v)
-	for i := range fp.ParamFlags {
+	funcProps.ParamFlags = make([]ParamPropBits, v)
+	for i := range funcProps.ParamFlags {
 		v, sl = readULEB128(sl)
-		fp.ParamFlags[i] = ParamPropBits(v)
+		funcProps.ParamFlags[i] = ParamPropBits(v)
 	}
 	v, sl = readULEB128(sl)
-	fp.ResultFlags = make([]ResultPropBits, v)
-	for i := range fp.ResultFlags {
+	funcProps.ResultFlags = make([]ResultPropBits, v)
+	for i := range funcProps.ResultFlags {
 		v, sl = readULEB128(sl)
-		fp.ResultFlags[i] = ResultPropBits(v)
+		funcProps.ResultFlags[i] = ResultPropBits(v)
 	}
-	return &fp
+	return &funcProps
 }
 
 func readULEB128(sl []byte) (value uint64, rsl []byte) {

src/cmd/compile/internal/inline/inlheur/testdata/dumpscores.go

@@ -0,0 +1,45 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dumpscores
+
+var G int
+
+func inlinable(x int, f func(int) int) int {
+	if x != 0 {
+		return 1
+	}
+	G += noninl(x)
+	return f(x)
+}
+
+func inlinable2(x int) int {
+	return noninl(-x)
+}
+
+//go:noinline
+func noninl(x int) int {
+	return x + 1
+}
+
+func tooLargeToInline(x int) int {
+	if x > 101 {
+		// Drive up the cost of inlining this func over the
+		// regular threshold.
+		return big(big(big(big(big(G + x)))))
+	}
+	if x < 100 {
+		// make sure this callsite is scored properly
+		G += inlinable(101, inlinable2)
+		if G == 101 {
+			return 0
+		}
+		panic(inlinable2(3))
+	}
+	return G
+}
+
+func big(q int) int {
+	return noninl(q) + noninl(-q)
+}

src/cmd/compile/internal/inline/inlheur/testdata/props/acrosscall.go

@@ -12,7 +12,7 @@ package params
 // ParamFlags
 //   0 ParamFeedsIndirectCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[8],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[8],"ResultFlags":null}
 // callsite: acrosscall.go:20:12|0 flagstr "" flagval 0 score 60 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -24,7 +24,7 @@ func T_feeds_indirect_call_via_call_toplevel(f func(int)) {
 // ParamFlags
 //   0 ParamMayFeedIndirectCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[16],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[16],"ResultFlags":null}
 // callsite: acrosscall.go:33:13|0 flagstr "" flagval 0 score 60 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -38,7 +38,7 @@ func T_feeds_indirect_call_via_call_conditional(f func(int)) {
 // ParamFlags
 //   0 ParamMayFeedIndirectCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[16],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[16],"ResultFlags":null}
 // callsite: acrosscall.go:46:23|0 flagstr "" flagval 0 score 64 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -50,7 +50,7 @@ func T_feeds_conditional_indirect_call_via_call_toplevel(f func(int)) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // callsite: acrosscall.go:58:9|0 flagstr "" flagval 0 score 8 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -62,7 +62,7 @@ func T_feeds_if_via_call(x int) {
 // ParamFlags
 //   0 ParamMayFeedIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[64],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[64],"ResultFlags":null}
 // callsite: acrosscall.go:71:10|0 flagstr "" flagval 0 score 8 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -76,7 +76,7 @@ func T_feeds_if_via_call_conditional(x int) {
 // ParamFlags
 //   0 ParamMayFeedIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[64],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[64],"ResultFlags":null}
 // callsite: acrosscall.go:84:20|0 flagstr "" flagval 0 score 12 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
@@ -84,63 +84,61 @@ func T_feeds_conditional_if_via_call(x int) {
 	feedsifconditional(x)
 }
 
-// acrosscall.go T_multifeeds 98 0 1
+// acrosscall.go T_multifeeds1 97 0 1
 // ParamFlags
 //   0 ParamFeedsIndirectCall|ParamMayFeedIndirectCall
-//   1 ParamFeedsIndirectCall
+//   1 ParamNoInfo
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[24,8],"ResultFlags":[]}
-// callsite: acrosscall.go:100:23|1 flagstr "" flagval 0 score 64 mask 0 maskstr ""
-// callsite: acrosscall.go:101:12|2 flagstr "" flagval 0 score 60 mask 0 maskstr ""
-// callsite: acrosscall.go:99:12|0 flagstr "" flagval 0 score 60 mask 0 maskstr ""
+// {"Flags":0,"ParamFlags":[24,0],"ResultFlags":null}
+// callsite: acrosscall.go:98:12|0 flagstr "" flagval 0 score 60 mask 0 maskstr ""
+// callsite: acrosscall.go:99:23|1 flagstr "" flagval 0 score 64 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
-func T_multifeeds(f1, f2 func(int)) {
+func T_multifeeds1(f1, f2 func(int)) {
 	callsparam(f1)
 	callsparamconditional(f1)
-	callsparam(f2)
 }
 
-// acrosscall.go T_acrosscall_returnsconstant 112 0 1
+// acrosscall.go T_acrosscall_returnsconstant 110 0 1
 // ResultFlags
 //   0 ResultAlwaysSameConstant
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[8]}
-// callsite: acrosscall.go:113:24|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[8]}
+// callsite: acrosscall.go:111:24|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_acrosscall_returnsconstant() int {
 	return returnsconstant()
 }
 
-// acrosscall.go T_acrosscall_returnsmem 124 0 1
+// acrosscall.go T_acrosscall_returnsmem 122 0 1
 // ResultFlags
 //   0 ResultIsAllocatedMem
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[2]}
-// callsite: acrosscall.go:125:19|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[2]}
+// callsite: acrosscall.go:123:19|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_acrosscall_returnsmem() *int {
 	return returnsmem()
 }
 
-// acrosscall.go T_acrosscall_returnscci 136 0 1
+// acrosscall.go T_acrosscall_returnscci 134 0 1
 // ResultFlags
 //   0 ResultIsConcreteTypeConvertedToInterface
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[4]}
-// callsite: acrosscall.go:137:19|0 flagstr "" flagval 0 score 7 mask 0 maskstr ""
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[4]}
+// callsite: acrosscall.go:135:19|0 flagstr "" flagval 0 score 7 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_acrosscall_returnscci() I {
 	return returnscci()
 }
 
-// acrosscall.go T_acrosscall_multiret 146 0 1
+// acrosscall.go T_acrosscall_multiret 144 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: acrosscall.go:148:25|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// callsite: acrosscall.go:146:25|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_acrosscall_multiret(q int) int {
@@ -150,11 +148,11 @@ func T_acrosscall_multiret(q int) int {
 	return 0
 }
 
-// acrosscall.go T_acrosscall_multiret2 160 0 1
+// acrosscall.go T_acrosscall_multiret2 158 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: acrosscall.go:162:25|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
-// callsite: acrosscall.go:164:25|1 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// callsite: acrosscall.go:160:25|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// callsite: acrosscall.go:162:25|1 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_acrosscall_multiret2(q int) int {

src/cmd/compile/internal/inline/inlheur/testdata/props/calls.go

@@ -12,7 +12,7 @@ import "os"
 
 // calls.go T_call_in_panic_arg 19 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // callsite: calls.go:21:15|0 flagstr "CallSiteOnPanicPath" flagval 2 score 42 mask 1 maskstr "panicPathAdj"
 // <endcallsites>
 // <endfuncpreamble>
@@ -24,7 +24,7 @@ func T_call_in_panic_arg(x int) {
 
 // calls.go T_calls_in_loops 32 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
 // callsite: calls.go:34:9|0 flagstr "CallSiteInLoop" flagval 1 score -3 mask 4 maskstr "inLoopAdj"
 // callsite: calls.go:37:9|1 flagstr "CallSiteInLoop" flagval 1 score -3 mask 4 maskstr "inLoopAdj"
 // <endcallsites>
@@ -40,7 +40,7 @@ func T_calls_in_loops(x int, q []string) {
 
 // calls.go T_calls_in_pseudo_loop 48 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
 // callsite: calls.go:50:9|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // callsite: calls.go:54:9|1 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
@@ -58,9 +58,9 @@ func T_calls_in_pseudo_loop(x int, q []string) {
 
 // calls.go T_calls_on_panic_paths 67 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
-// callsite: calls.go:69:9|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
-// callsite: calls.go:73:9|1 flagstr "" flagval 0 score 2 mask 0 maskstr ""
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
+// callsite: calls.go:69:9|0 flagstr "CallSiteOnPanicPath" flagval 2 score 42 mask 1 maskstr "panicPathAdj"
+// callsite: calls.go:73:9|1 flagstr "CallSiteOnPanicPath" flagval 2 score 42 mask 1 maskstr "panicPathAdj"
 // callsite: calls.go:77:12|2 flagstr "CallSiteOnPanicPath" flagval 2 score 102 mask 1 maskstr "panicPathAdj"
 // <endcallsites>
 // <endfuncpreamble>
@@ -83,11 +83,11 @@ func T_calls_on_panic_paths(x int, q []string) {
 //   0 ParamFeedsIfOrSwitch|ParamMayFeedIfOrSwitch
 //   1 ParamNoInfo
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[96,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[96,0],"ResultFlags":null}
 // callsite: calls.go:103:9|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // callsite: calls.go:112:9|1 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // callsite: calls.go:115:9|2 flagstr "" flagval 0 score 2 mask 0 maskstr ""
-// callsite: calls.go:119:12|3 flagstr "" flagval 0 score 62 mask 0 maskstr ""
+// callsite: calls.go:119:12|3 flagstr "CallSiteOnPanicPath" flagval 2 score 102 mask 1 maskstr "panicPathAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_calls_not_on_panic_paths(x int, q []string) {
@@ -122,7 +122,7 @@ func T_calls_not_on_panic_paths(x int, q []string) {
 
 // calls.go init.0 129 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":null}
 // callsite: calls.go:130:16|0 flagstr "CallSiteInInitFunc" flagval 4 score 22 mask 2 maskstr "initFuncAdj"
 // <endcallsites>
 // <endfuncpreamble>
@@ -130,20 +130,21 @@ func init() {
 	println(callee(5))
 }
 
-// calls.go T_pass_inlinable_func_to_param_feeding_indirect_call 139 0 1
+// calls.go T_pass_inlinable_func_to_param_feeding_indirect_call 140 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: calls.go:140:19|0 flagstr "" flagval 0 score 16 mask 512 maskstr "passInlinableFuncToIndCallAdj"
+// callsite: calls.go:141:19|0 flagstr "" flagval 0 score 16 mask 512 maskstr "passInlinableFuncToIndCallAdj"
+// callsite: calls.go:141:19|calls.go:232:10|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_pass_inlinable_func_to_param_feeding_indirect_call(x int) int {
 	return callsParam(x, callee)
 }
 
-// calls.go T_pass_noninlinable_func_to_param_feeding_indirect_call 149 0 1
+// calls.go T_pass_noninlinable_func_to_param_feeding_indirect_call 150 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: calls.go:152:19|0 flagstr "" flagval 0 score 36 mask 128 maskstr "passFuncToIndCallAdj"
+// callsite: calls.go:153:19|0 flagstr "" flagval 0 score 36 mask 128 maskstr "passFuncToIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_pass_noninlinable_func_to_param_feeding_indirect_call(x int) int {
@@ -152,30 +153,65 @@ func T_pass_noninlinable_func_to_param_feeding_indirect_call(x int) int {
 	return callsParam(x, calleeNoInline)
 }
 
-// calls.go T_pass_inlinable_func_to_param_feeding_nested_indirect_call 163 0 1
+// calls.go T_pass_inlinable_func_to_param_feeding_nested_indirect_call 165 0 1
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[32],"ResultFlags":[0]}
-// callsite: calls.go:164:25|0 flagstr "" flagval 0 score 27 mask 1024 maskstr "passInlinableFuncToNestedIndCallAdj"
+// callsite: calls.go:166:25|0 flagstr "" flagval 0 score 27 mask 1024 maskstr "passInlinableFuncToNestedIndCallAdj"
+// callsite: calls.go:166:25|calls.go:237:11|0 flagstr "" flagval 0 score 2 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
 func T_pass_inlinable_func_to_param_feeding_nested_indirect_call(x int) int {
 	return callsParamNested(x, callee)
 }
 
-// calls.go T_pass_noninlinable_func_to_param_feeding_nested_indirect_call 175 0 1
+// calls.go T_pass_noninlinable_func_to_param_feeding_nested_indirect_call 177 0 1
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[32],"ResultFlags":[0]}
-// callsite: calls.go:176:25|0 flagstr "" flagval 0 score 47 mask 256 maskstr "passFuncToNestedIndCallAdj"
+// callsite: calls.go:178:25|0 flagstr "" flagval 0 score 47 mask 256 maskstr "passFuncToNestedIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_pass_noninlinable_func_to_param_feeding_nested_indirect_call(x int) int {
 	return callsParamNested(x, calleeNoInline)
 }
 
+// calls.go T_call_scoring_in_noninlinable_func 195 0 1
+// <endpropsdump>
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[0]}
+// callsite: calls.go:209:14|0 flagstr "CallSiteOnPanicPath" flagval 2 score 42 mask 1 maskstr "panicPathAdj"
+// callsite: calls.go:210:15|1 flagstr "CallSiteOnPanicPath" flagval 2 score 42 mask 1 maskstr "panicPathAdj"
+// callsite: calls.go:212:19|2 flagstr "" flagval 0 score 16 mask 512 maskstr "passInlinableFuncToIndCallAdj"
+// callsite: calls.go:212:19|calls.go:232:10|0 flagstr "" flagval 0 score 4 mask 0 maskstr ""
+// <endcallsites>
+// <endfuncpreamble>
+// calls.go T_call_scoring_in_noninlinable_func.func1 212 0 1
+// <endpropsdump>
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
+// <endcallsites>
+// <endfuncpreamble>
+func T_call_scoring_in_noninlinable_func(x int, sl []int) int {
+	if x == 101 {
+		// Drive up the cost of inlining this funcfunc over the
+		// regular threshold.
+		for i := 0; i < 10; i++ {
+			for j := 0; j < i; j++ {
+				sl = append(sl, append(sl, append(sl, append(sl, x)...)...)...)
+				sl = append(sl, sl[0], sl[1], sl[2])
+				x += calleeNoInline(x)
+			}
+		}
+	}
+	if x < 100 {
+		// make sure this callsite is scored properly
+		G += callee(101)
+		panic(callee(x))
+	}
+	return callsParam(x, func(y int) int { return y + x })
+}
+
 var G int
 
 func callee(x int) int {

src/cmd/compile/internal/inline/inlheur/testdata/props/funcflags.go

@@ -14,7 +14,7 @@ import "os"
 // funcflags.go T_simple 20 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":null,"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_simple() {
@@ -26,7 +26,7 @@ func T_simple() {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_nested(x int) {
@@ -40,7 +40,7 @@ func T_nested(x int) {
 // funcflags.go T_block1 46 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_block1(x int) {
@@ -54,7 +54,7 @@ func T_block1(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_block2(x int) {
@@ -69,7 +69,7 @@ func T_block2(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_switches1(x int) {
@@ -86,7 +86,7 @@ func T_switches1(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_switches1a(x int) {
@@ -100,7 +100,7 @@ func T_switches1a(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_switches2(x int) {
@@ -117,7 +117,7 @@ func T_switches2(x int) {
 
 // funcflags.go T_switches3 123 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_switches3(x interface{}) {
@@ -132,7 +132,7 @@ func T_switches3(x interface{}) {
 // funcflags.go T_switches4 138 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_switches4(x int) {
@@ -151,7 +151,7 @@ func T_switches4(x int) {
 
 // funcflags.go T_recov 157 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_recov(x int) {
@@ -163,7 +163,7 @@ func T_recov(x int) {
 // funcflags.go T_forloops1 169 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_forloops1(x int) {
@@ -174,7 +174,7 @@ func T_forloops1(x int) {
 
 // funcflags.go T_forloops2 180 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_forloops2(x int) {
@@ -189,7 +189,7 @@ func T_forloops2(x int) {
 
 // funcflags.go T_forloops3 195 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_forloops3(x int) {
@@ -209,7 +209,7 @@ func T_forloops3(x int) {
 
 // funcflags.go T_hasgotos 215 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_hasgotos(x int, y int) {
@@ -240,7 +240,7 @@ func T_hasgotos(x int, y int) {
 //   0 ParamMayFeedIfOrSwitch
 //   1 ParamNoInfo
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[64,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[64,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_break_with_label(x int, y int) {
@@ -262,7 +262,7 @@ lab1:
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_callsexit(x int) {
@@ -274,7 +274,7 @@ func T_callsexit(x int) {
 
 // funcflags.go T_exitinexpr 281 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // callsite: funcflags.go:286:18|0 flagstr "CallSiteOnPanicPath" flagval 2 score 102 mask 1 maskstr "panicPathAdj"
 // <endcallsites>
 // <endfuncpreamble>
@@ -291,7 +291,7 @@ func T_exitinexpr(x int) {
 // funcflags.go T_select_noreturn 297 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[0,0,0],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[0,0,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_select_noreturn(chi chan int, chf chan float32, p *int) {
@@ -327,7 +327,7 @@ func T_select_mayreturn(chi chan int, chf chan float32, p *int) int {
 // funcflags.go T_calls_callsexit 334 0 1
 // Flags FuncPropNeverReturns
 // <endpropsdump>
-// {"Flags":1,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":1,"ParamFlags":[0],"ResultFlags":null}
 // callsite: funcflags.go:335:15|0 flagstr "CallSiteOnPanicPath" flagval 2 score 102 mask 1 maskstr "panicPathAdj"
 // <endcallsites>
 // <endfuncpreamble>

src/cmd/compile/internal/inline/inlheur/testdata/props/params.go

@@ -14,7 +14,7 @@ import "os"
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_simple(x int) {
@@ -29,7 +29,7 @@ func T_feeds_if_simple(x int) {
 //   0 ParamMayFeedIfOrSwitch
 //   1 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[64,32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[64,32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_nested(x, y int) {
@@ -45,7 +45,7 @@ func T_feeds_if_nested(x, y int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_pointer(xp *int) {
@@ -60,7 +60,7 @@ func T_feeds_if_pointer(xp *int) {
 //   0 ParamFeedsIfOrSwitch
 //   1 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32,32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32,32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func (r T) T_feeds_if_simple_method(x int) {
@@ -80,7 +80,7 @@ func (r T) T_feeds_if_simple_method(x int) {
 //   2 ParamNoInfo
 //   3 ParamNoInfo
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,32,0,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0,32,0,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_blanks(_ string, x int, _ bool, _ bool) {
@@ -95,7 +95,7 @@ func T_feeds_if_blanks(_ string, x int, _ bool, _ bool) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_with_copy(x int) {
@@ -109,7 +109,7 @@ func T_feeds_if_with_copy(x int) {
 
 // params.go T_feeds_if_with_copy_expr 115 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_with_copy_expr(x int) {
@@ -125,7 +125,7 @@ func T_feeds_if_with_copy_expr(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_switch(x int) {
@@ -140,7 +140,7 @@ func T_feeds_switch(x int) {
 
 // params.go T_feeds_if_toocomplex 146 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_toocomplex(x int, y int) {
@@ -155,7 +155,7 @@ func T_feeds_if_toocomplex(x int, y int) {
 
 // params.go T_feeds_if_redefined 161 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_redefined(x int) {
@@ -169,7 +169,7 @@ func T_feeds_if_redefined(x int) {
 
 // params.go T_feeds_if_redefined2 175 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_redefined2(x int) {
@@ -190,7 +190,7 @@ func T_feeds_if_redefined2(x int) {
 //   0 ParamFeedsIfOrSwitch
 //   1 ParamNoInfo
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32,0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32,0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_multi_if(x int, y int) {
@@ -210,7 +210,7 @@ func T_feeds_multi_if(x int, y int) {
 
 // params.go T_feeds_if_redefined_indirectwrite 216 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_redefined_indirectwrite(x int) {
@@ -225,7 +225,7 @@ func T_feeds_if_redefined_indirectwrite(x int) {
 
 // params.go T_feeds_if_redefined_indirectwrite_copy 231 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_redefined_indirectwrite_copy(x int) {
@@ -245,7 +245,7 @@ func T_feeds_if_redefined_indirectwrite_copy(x int) {
 // ParamFlags
 //   0 ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[32],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[32],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_expr1(x int) {
@@ -256,7 +256,7 @@ func T_feeds_if_expr1(x int) {
 
 // params.go T_feeds_if_expr2 262 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_expr2(x int) {
@@ -267,7 +267,7 @@ func T_feeds_if_expr2(x int) {
 
 // params.go T_feeds_if_expr3 273 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_expr3(x int) {
@@ -293,7 +293,7 @@ func T_feeds_if_shift_may_panic(x int) *int {
 
 // params.go T_feeds_if_maybe_divide_by_zero 299 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_if_maybe_divide_by_zero(x int) {
@@ -307,7 +307,7 @@ func T_feeds_if_maybe_divide_by_zero(x int) {
 // ParamFlags
 //   0 ParamMayFeedIndirectCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[16],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[16],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_indcall(x func()) {
@@ -320,7 +320,7 @@ func T_feeds_indcall(x func()) {
 // ParamFlags
 //   0 ParamMayFeedIndirectCall|ParamFeedsIfOrSwitch
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[48],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[48],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_indcall_and_if(x func()) {
@@ -333,7 +333,7 @@ func T_feeds_indcall_and_if(x func()) {
 // ParamFlags
 //   0 ParamFeedsIndirectCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[8],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[8],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_indcall_with_copy(x func()) {
@@ -348,7 +348,7 @@ func T_feeds_indcall_with_copy(x func()) {
 // ParamFlags
 //   0 ParamFeedsInterfaceMethodCall
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[2],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":[2],"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_feeds_interface_method_call(i I) {

src/cmd/compile/internal/inline/inlheur/testdata/props/returns.go

@@ -15,7 +15,7 @@ import "unsafe"
 // ResultFlags
 //   0 ResultIsAllocatedMem
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[2]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[2]}
 // <endcallsites>
 // <endfuncpreamble>
 func T_simple_allocmem() *Bar {
@@ -66,7 +66,7 @@ func T_allocmem_three_returns(x int) []*Bar {
 // ResultFlags
 //   0 ResultAlwaysSameConstant
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[8]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[8]}
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_nil() *Bar {
@@ -247,7 +247,7 @@ func T_return_concrete_type_to_itf_mixed(x, y int) Itf {
 // ResultFlags
 //   0 ResultAlwaysSameInlinableFunc
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[32]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[32]}
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_same_func() func(int) int {
@@ -260,7 +260,7 @@ func T_return_same_func() func(int) int {
 
 // returns.go T_return_different_funcs 266 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[0]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[0]}
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_different_funcs() func(int) int {
@@ -275,7 +275,7 @@ func T_return_different_funcs() func(int) int {
 // ResultFlags
 //   0 ResultAlwaysSameInlinableFunc
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[32]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[32]}
 // <endcallsites>
 // <endfuncpreamble>
 // returns.go T_return_same_closure.func1 287 0 1
@@ -294,7 +294,7 @@ func T_return_same_closure() func(int) int {
 
 // returns.go T_return_different_closures 312 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[0]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":[0]}
 // <endcallsites>
 // <endfuncpreamble>
 // returns.go T_return_different_closures.func1 313 0 1
@@ -318,21 +318,22 @@ func T_return_different_closures() func(int) int {
 	}
 }
 
-// returns.go T_return_noninlinable 338 0 1
+// returns.go T_return_noninlinable 339 0 1
 // ResultFlags
 //   0 ResultAlwaysSameFunc
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[16]}
 // <endcallsites>
 // <endfuncpreamble>
-// returns.go T_return_noninlinable.func1 339 0 1
+// returns.go T_return_noninlinable.func1 340 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
+// callsite: returns.go:343:4|0 flagstr "" flagval 0 score 4 mask 0 maskstr ""
 // <endcallsites>
 // <endfuncpreamble>
-// returns.go T_return_noninlinable.func1.1 340 0 1
+// returns.go T_return_noninlinable.func1.1 341 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[]}
+// {"Flags":0,"ParamFlags":null,"ResultFlags":null}
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_noninlinable(x int) func(int) int {

src/cmd/compile/internal/inline/inlheur/testdata/props/returns2.go

@@ -11,8 +11,8 @@ package returns2
 
 // returns2.go T_return_feeds_iface_call 18 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[]}
-// callsite: returns2.go:19:13|0 flagstr "" flagval 0 score -4 mask 2080 maskstr "passConcreteToItfCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":null,"ResultFlags":null}
+// callsite: returns2.go:19:13|0 flagstr "" flagval 0 score 1 mask 16384 maskstr "returnFeedsConcreteToInterfaceCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_feeds_iface_call() {
@@ -22,8 +22,8 @@ func T_return_feeds_iface_call() {
 
 // returns2.go T_multi_return_feeds_iface_call 29 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[],"ResultFlags":[]}
-// callsite: returns2.go:30:20|0 flagstr "" flagval 0 score -2 mask 2080 maskstr "passConcreteToItfCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":null,"ResultFlags":null}
+// callsite: returns2.go:30:20|0 flagstr "" flagval 0 score 3 mask 16384 maskstr "returnFeedsConcreteToInterfaceCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_multi_return_feeds_iface_call() {
@@ -33,13 +33,13 @@ func T_multi_return_feeds_iface_call() {
 
 // returns2.go T_returned_inlinable_func_feeds_indirect_call 41 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
-// callsite: returns2.go:42:18|0 flagstr "" flagval 0 score -43 mask 2560 maskstr "passInlinableFuncToIndCallAdj|callResultRescoreAdj"
-// callsite: returns2.go:44:20|1 flagstr "" flagval 0 score -28 mask 2560 maskstr "passInlinableFuncToIndCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
+// callsite: returns2.go:42:18|0 flagstr "" flagval 0 score -51 mask 8200 maskstr "passConstToIfAdj|returnFeedsInlinableFuncToIndCallAdj"
+// callsite: returns2.go:44:20|1 flagstr "" flagval 0 score -23 mask 8192 maskstr "returnFeedsInlinableFuncToIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_returned_inlinable_func_feeds_indirect_call(q int) {
-	f := returnsFunc()
+	f := returnsFunc(10)
 	f(q)
 	f2 := returnsFunc2()
 	f2(q)
@@ -47,8 +47,8 @@ func T_returned_inlinable_func_feeds_indirect_call(q int) {
 
 // returns2.go T_returned_noninlineable_func_feeds_indirect_call 54 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
-// callsite: returns2.go:55:30|0 flagstr "" flagval 0 score -23 mask 2176 maskstr "passFuncToIndCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
+// callsite: returns2.go:55:30|0 flagstr "" flagval 0 score -23 mask 4096 maskstr "returnFeedsFuncToIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_returned_noninlineable_func_feeds_indirect_call(q int) {
@@ -58,8 +58,8 @@ func T_returned_noninlineable_func_feeds_indirect_call(q int) {
 
 // returns2.go T_multi_return_feeds_indirect_call 65 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0],"ResultFlags":[]}
-// callsite: returns2.go:66:29|0 flagstr "" flagval 0 score -26 mask 2560 maskstr "passInlinableFuncToIndCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":[0],"ResultFlags":null}
+// callsite: returns2.go:66:29|0 flagstr "" flagval 0 score -21 mask 8192 maskstr "returnFeedsInlinableFuncToIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_multi_return_feeds_indirect_call(q int) {
@@ -70,7 +70,7 @@ func T_multi_return_feeds_indirect_call(q int) {
 // returns2.go T_return_feeds_ifswitch 76 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: returns2.go:77:14|0 flagstr "" flagval 0 score 5 mask 2056 maskstr "passConstToIfAdj|callResultRescoreAdj"
+// callsite: returns2.go:77:14|0 flagstr "" flagval 0 score 10 mask 2048 maskstr "returnFeedsConstToIfAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_return_feeds_ifswitch(q int) int {
@@ -87,7 +87,7 @@ func T_return_feeds_ifswitch(q int) int {
 // returns2.go T_multi_return_feeds_ifswitch 93 0 1
 // <endpropsdump>
 // {"Flags":0,"ParamFlags":[0],"ResultFlags":[0]}
-// callsite: returns2.go:94:21|0 flagstr "" flagval 0 score 4 mask 2056 maskstr "passConstToIfAdj|callResultRescoreAdj"
+// callsite: returns2.go:94:21|0 flagstr "" flagval 0 score 9 mask 2048 maskstr "returnFeedsConstToIfAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_multi_return_feeds_ifswitch(q int) int {
@@ -125,20 +125,20 @@ func T_two_calls_feed_ifswitch(q int) int {
 
 // returns2.go T_chained_indirect_call 132 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
-// callsite: returns2.go:135:18|0 flagstr "" flagval 0 score -43 mask 2560 maskstr "passInlinableFuncToIndCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
+// callsite: returns2.go:135:18|0 flagstr "" flagval 0 score -31 mask 8192 maskstr "returnFeedsInlinableFuncToIndCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_chained_indirect_call(x, y int) {
 	// Here 'returnsFunc' returns an inlinable func that feeds
 	// directly into a call (no named intermediate).
-	G += returnsFunc()(x + y)
+	G += returnsFunc(x - y)(x + y)
 }
 
 // returns2.go T_chained_conc_iface_call 144 0 1
 // <endpropsdump>
-// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":[]}
-// callsite: returns2.go:148:8|0 flagstr "" flagval 0 score -4 mask 2080 maskstr "passConcreteToItfCallAdj|callResultRescoreAdj"
+// {"Flags":0,"ParamFlags":[0,0],"ResultFlags":null}
+// callsite: returns2.go:148:8|0 flagstr "" flagval 0 score 1 mask 16384 maskstr "returnFeedsConcreteToInterfaceCallAdj"
 // <endcallsites>
 // <endfuncpreamble>
 func T_chained_conc_iface_call(x, y int) {
@@ -148,7 +148,10 @@ func T_chained_conc_iface_call(x, y int) {
 	newBar(10).Plark().Plark()
 }
 
-func returnsFunc() func(int) int {
+func returnsFunc(x int) func(int) int {
+	if x < 0 {
+		G++
+	}
 	return adder
 }
 

src/cmd/compile/internal/inline/interleaved/interleaved.go

@@ -0,0 +1,132 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package interleaved implements the interleaved devirtualization and
+// inlining pass.
+package interleaved
+
+import (
+	"cmd/compile/internal/base"
+	"cmd/compile/internal/devirtualize"
+	"cmd/compile/internal/inline"
+	"cmd/compile/internal/inline/inlheur"
+	"cmd/compile/internal/ir"
+	"cmd/compile/internal/pgo"
+	"cmd/compile/internal/typecheck"
+	"fmt"
+)
+
+// DevirtualizeAndInlinePackage interleaves devirtualization and inlining on
+// all functions within pkg.
+func DevirtualizeAndInlinePackage(pkg *ir.Package, profile *pgo.Profile) {
+	if profile != nil && base.Debug.PGODevirtualize > 0 {
+		// TODO(mdempsky): Integrate into DevirtualizeAndInlineFunc below.
+		ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) {
+			for _, fn := range list {
+				devirtualize.ProfileGuided(fn, profile)
+			}
+		})
+		ir.CurFunc = nil
+	}
+
+	if base.Flag.LowerL != 0 {
+		inlheur.SetupScoreAdjustments()
+	}
+
+	var inlProfile *pgo.Profile // copy of profile for inlining
+	if base.Debug.PGOInline != 0 {
+		inlProfile = profile
+	}
+	if inlProfile != nil {
+		inline.PGOInlinePrologue(inlProfile, pkg.Funcs)
+	}
+
+	ir.VisitFuncsBottomUp(pkg.Funcs, func(funcs []*ir.Func, recursive bool) {
+		// We visit functions within an SCC in fairly arbitrary order,
+		// so by computing inlinability for all functions in the SCC
+		// before performing any inlining, the results are less
+		// sensitive to the order within the SCC (see #58905 for an
+		// example).
+
+		// First compute inlinability for all functions in the SCC ...
+		inline.CanInlineSCC(funcs, recursive, inlProfile)
+
+		// ... then make a second pass to do devirtualization and inlining
+		// of calls.
+		for _, fn := range funcs {
+			DevirtualizeAndInlineFunc(fn, inlProfile)
+		}
+	})
+
+	if base.Flag.LowerL != 0 {
+		// Perform a garbage collection of hidden closures functions that
+		// are no longer reachable from top-level functions following
+		// inlining. See #59404 and #59638 for more context.
+		inline.GarbageCollectUnreferencedHiddenClosures()
+
+		if base.Debug.DumpInlFuncProps != "" {
+			inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps)
+		}
+		if inlheur.Enabled() {
+			inline.PostProcessCallSites(inlProfile)
+			inlheur.TearDown()
+		}
+	}
+}
+
+// DevirtualizeAndInlineFunc interleaves devirtualization and inlining
+// on a single function.
+func DevirtualizeAndInlineFunc(fn *ir.Func, profile *pgo.Profile) {
+	ir.WithFunc(fn, func() {
+		if base.Flag.LowerL != 0 {
+			if inlheur.Enabled() && !fn.Wrapper() {
+				inlheur.ScoreCalls(fn)
+				defer inlheur.ScoreCallsCleanup()
+			}
+			if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
+				inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps)
+			}
+		}
+
+		bigCaller := base.Flag.LowerL != 0 && inline.IsBigFunc(fn)
+		if bigCaller && base.Flag.LowerM > 1 {
+			fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
+		}
+
+		// Walk fn's body and apply devirtualization and inlining.
+		var inlCalls []*ir.InlinedCallExpr
+		var edit func(ir.Node) ir.Node
+		edit = func(n ir.Node) ir.Node {
+			switch n := n.(type) {
+			case *ir.TailCallStmt:
+				n.Call.NoInline = true // can't inline yet
+			}
+
+			ir.EditChildren(n, edit)
+
+			if call, ok := n.(*ir.CallExpr); ok {
+				devirtualize.StaticCall(call)
+
+				if inlCall := inline.TryInlineCall(fn, call, bigCaller, profile); inlCall != nil {
+					inlCalls = append(inlCalls, inlCall)
+					n = inlCall
+				}
+			}
+
+			return n
+		}
+		ir.EditChildren(fn, edit)
+
+		// If we inlined any calls, we want to recursively visit their
+		// bodies for further devirtualization and inlining. However, we
+		// need to wait until *after* the original function body has been
+		// expanded, or else inlCallee can have false positives (e.g.,
+		// #54632).
+		for len(inlCalls) > 0 {
+			call := inlCalls[0]
+			inlCalls = inlCalls[1:]
+			ir.EditChildren(call, edit)
+		}
+	})
+}

src/cmd/compile/internal/ir/check_reassign_no.go

@@ -0,0 +1,9 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !checknewoldreassignment
+
+package ir
+
+const consistencyCheckEnabled = false

src/cmd/compile/internal/ir/check_reassign_yes.go

@@ -0,0 +1,9 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build checknewoldreassignment
+
+package ir
+
+const consistencyCheckEnabled = true

src/cmd/compile/internal/ir/expr.go

@@ -54,7 +54,7 @@ func (n *miniExpr) Init() Nodes           { return n.init }
 func (n *miniExpr) PtrInit() *Nodes       { return &n.init }
 func (n *miniExpr) SetInit(x Nodes)       { n.init = x }
 
-// An AddStringExpr is a string concatenation Expr[0] + Exprs[1] + ... + Expr[len(Expr)-1].
+// An AddStringExpr is a string concatenation List[0] + List[1] + ... + List[len(List)-1].
 type AddStringExpr struct {
 	miniExpr
 	List     Nodes
@@ -190,7 +190,8 @@ type CallExpr struct {
 	RType     Node    `mknode:"-"` // see reflectdata/helpers.go
 	KeepAlive []*Name // vars to be kept alive until call returns
 	IsDDD     bool
-	NoInline  bool
+	GoDefer   bool // whether this call is part of a go or defer statement
+	NoInline  bool // whether this call must not be inlined
 }
 
 func NewCallExpr(pos src.XPos, op Op, fun Node, args []Node) *CallExpr {
@@ -349,7 +350,7 @@ func NewKeyExpr(pos src.XPos, key, value Node) *KeyExpr {
 	return n
 }
 
-// A StructKeyExpr is an Field: Value composite literal key.
+// A StructKeyExpr is a Field: Value composite literal key.
 type StructKeyExpr struct {
 	miniExpr
 	Field *types.Field
@@ -922,6 +923,8 @@ FindRHS:
 // NB: global variables are always considered to be re-assigned.
 // TODO: handle initial declaration not including an assignment and
 // followed by a single assignment?
+// NOTE: any changes made here should also be made in the corresponding
+// code in the ReassignOracle.Init method.
 func Reassigned(name *Name) bool {
 	if name.Op() != ONAME {
 		base.Fatalf("reassigned %v", name)

src/cmd/compile/internal/ir/func.go

@@ -126,7 +126,7 @@ type Func struct {
 	NumDefers  int32 // number of defer calls in the function
 	NumReturns int32 // number of explicit returns in the function
 
-	// nwbrCalls records the LSyms of functions called by this
+	// NWBRCalls records the LSyms of functions called by this
 	// function for go:nowritebarrierrec analysis. Only filled in
 	// if nowritebarrierrecCheck != nil.
 	NWBRCalls *[]SymAndPos
@@ -505,6 +505,61 @@ func IsFuncPCIntrinsic(n *CallExpr) bool {
 		fn.Pkg.Path == "internal/abi"
 }
 
+// IsIfaceOfFunc inspects whether n is an interface conversion from a direct
+// reference of a func. If so, it returns referenced Func; otherwise nil.
+//
+// This is only usable before walk.walkConvertInterface, which converts to an
+// OMAKEFACE.
+func IsIfaceOfFunc(n Node) *Func {
+	if n, ok := n.(*ConvExpr); ok && n.Op() == OCONVIFACE {
+		if name, ok := n.X.(*Name); ok && name.Op() == ONAME && name.Class == PFUNC {
+			return name.Func
+		}
+	}
+	return nil
+}
+
+// FuncPC returns a uintptr-typed expression that evaluates to the PC of a
+// function as uintptr, as returned by internal/abi.FuncPC{ABI0,ABIInternal}.
+//
+// n should be a Node of an interface type, as is passed to
+// internal/abi.FuncPC{ABI0,ABIInternal}.
+//
+// TODO(prattmic): Since n is simply an interface{} there is no assertion that
+// it is actually a function at all. Perhaps we should emit a runtime type
+// assertion?
+func FuncPC(pos src.XPos, n Node, wantABI obj.ABI) Node {
+	if !n.Type().IsInterface() {
+		base.ErrorfAt(pos, 0, "internal/abi.FuncPC%s expects an interface value, got %v", wantABI, n.Type())
+	}
+
+	if fn := IsIfaceOfFunc(n); fn != nil {
+		name := fn.Nname
+		abi := fn.ABI
+		if abi != wantABI {
+			base.ErrorfAt(pos, 0, "internal/abi.FuncPC%s expects an %v function, %s is defined as %v", wantABI, wantABI, name.Sym().Name, abi)
+		}
+		var e Node = NewLinksymExpr(pos, name.Sym().LinksymABI(abi), types.Types[types.TUINTPTR])
+		e = NewAddrExpr(pos, e)
+		e.SetType(types.Types[types.TUINTPTR].PtrTo())
+		e = NewConvExpr(pos, OCONVNOP, types.Types[types.TUINTPTR], e)
+		e.SetTypecheck(1)
+		return e
+	}
+	// fn is not a defined function. It must be ABIInternal.
+	// Read the address from func value, i.e. *(*uintptr)(idata(fn)).
+	if wantABI != obj.ABIInternal {
+		base.ErrorfAt(pos, 0, "internal/abi.FuncPC%s does not accept func expression, which is ABIInternal", wantABI)
+	}
+	var e Node = NewUnaryExpr(pos, OIDATA, n)
+	e.SetType(types.Types[types.TUINTPTR].PtrTo())
+	e.SetTypecheck(1)
+	e = NewStarExpr(pos, e)
+	e.SetType(types.Types[types.TUINTPTR])
+	e.SetTypecheck(1)
+	return e
+}
+
 // DeclareParams creates Names for all of the parameters in fn's
 // signature and adds them to fn.Dcl.
 //

src/cmd/compile/internal/ir/reassign_consistency_check.go

@@ -0,0 +1,46 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+	"cmd/compile/internal/base"
+	"cmd/internal/src"
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// checkStaticValueResult compares the result from ReassignOracle.StaticValue
+// with the corresponding result from ir.StaticValue to make sure they agree.
+// This method is called only when turned on via build tag.
+func checkStaticValueResult(n Node, newres Node) {
+	oldres := StaticValue(n)
+	if oldres != newres {
+		base.Fatalf("%s: new/old static value disagreement on %v:\nnew=%v\nold=%v", fmtFullPos(n.Pos()), n, newres, oldres)
+	}
+}
+
+// checkStaticValueResult compares the result from ReassignOracle.Reassigned
+// with the corresponding result from ir.Reassigned to make sure they agree.
+// This method is called only when turned on via build tag.
+func checkReassignedResult(n *Name, newres bool) {
+	origres := Reassigned(n)
+	if newres != origres {
+		base.Fatalf("%s: new/old reassigned disagreement on %v (class %s) newres=%v oldres=%v", fmtFullPos(n.Pos()), n, n.Class.String(), newres, origres)
+	}
+}
+
+// fmtFullPos returns a verbose dump for pos p, including inlines.
+func fmtFullPos(p src.XPos) string {
+	var sb strings.Builder
+	sep := ""
+	base.Ctxt.AllPos(p, func(pos src.Pos) {
+		fmt.Fprintf(&sb, sep)
+		sep = "|"
+		file := filepath.Base(pos.Filename())
+		fmt.Fprintf(&sb, "%s:%d:%d", file, pos.Line(), pos.Col())
+	})
+	return sb.String()
+}

src/cmd/compile/internal/ir/reassignment.go

@@ -0,0 +1,205 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+	"cmd/compile/internal/base"
+)
+
+// A ReassignOracle efficiently answers queries about whether local
+// variables are reassigned. This helper works by looking for function
+// params and short variable declarations (e.g.
+// https://go.dev/ref/spec#Short_variable_declarations) that are
+// neither address taken nor subsequently re-assigned. It is intended
+// to operate much like "ir.StaticValue" and "ir.Reassigned", but in a
+// way that does just a single walk of the containing function (as
+// opposed to a new walk on every call).
+type ReassignOracle struct {
+	fn *Func
+	// maps candidate name to its defining assignment (or for
+	// for params, defining func).
+	singleDef map[*Name]Node
+}
+
+// Init initializes the oracle based on the IR in function fn, laying
+// the groundwork for future calls to the StaticValue and Reassigned
+// methods. If the fn's IR is subsequently modified, Init must be
+// called again.
+func (ro *ReassignOracle) Init(fn *Func) {
+	ro.fn = fn
+
+	// Collect candidate map. Start by adding function parameters
+	// explicitly.
+	ro.singleDef = make(map[*Name]Node)
+	sig := fn.Type()
+	numParams := sig.NumRecvs() + sig.NumParams()
+	for _, param := range fn.Dcl[:numParams] {
+		if IsBlank(param) {
+			continue
+		}
+		// For params, use func itself as defining node.
+		ro.singleDef[param] = fn
+	}
+
+	// Walk the function body to discover any locals assigned
+	// via ":=" syntax (e.g. "a := <expr>").
+	var findLocals func(n Node) bool
+	findLocals = func(n Node) bool {
+		if nn, ok := n.(*Name); ok {
+			if nn.Defn != nil && !nn.Addrtaken() && nn.Class == PAUTO {
+				ro.singleDef[nn] = nn.Defn
+			}
+		} else if nn, ok := n.(*ClosureExpr); ok {
+			Any(nn.Func, findLocals)
+		}
+		return false
+	}
+	Any(fn, findLocals)
+
+	outerName := func(x Node) *Name {
+		if x == nil {
+			return nil
+		}
+		n, ok := OuterValue(x).(*Name)
+		if ok {
+			return n.Canonical()
+		}
+		return nil
+	}
+
+	// pruneIfNeeded examines node nn appearing on the left hand side
+	// of assignment statement asn to see if it contains a reassignment
+	// to any nodes in our candidate map ro.singleDef; if a reassignment
+	// is found, the corresponding name is deleted from singleDef.
+	pruneIfNeeded := func(nn Node, asn Node) {
+		oname := outerName(nn)
+		if oname == nil {
+			return
+		}
+		defn, ok := ro.singleDef[oname]
+		if !ok {
+			return
+		}
+		// any assignment to a param invalidates the entry.
+		paramAssigned := oname.Class == PPARAM
+		// assignment to local ok iff assignment is its orig def.
+		localAssigned := (oname.Class == PAUTO && asn != defn)
+		if paramAssigned || localAssigned {
+			// We found an assignment to name N that doesn't
+			// correspond to its original definition; remove
+			// from candidates.
+			delete(ro.singleDef, oname)
+		}
+	}
+
+	// Prune away anything that looks assigned. This code modeled after
+	// similar code in ir.Reassigned; any changes there should be made
+	// here as well.
+	var do func(n Node) bool
+	do = func(n Node) bool {
+		switch n.Op() {
+		case OAS:
+			asn := n.(*AssignStmt)
+			pruneIfNeeded(asn.X, n)
+		case OAS2, OAS2FUNC, OAS2MAPR, OAS2DOTTYPE, OAS2RECV, OSELRECV2:
+			asn := n.(*AssignListStmt)
+			for _, p := range asn.Lhs {
+				pruneIfNeeded(p, n)
+			}
+		case OASOP:
+			asn := n.(*AssignOpStmt)
+			pruneIfNeeded(asn.X, n)
+		case ORANGE:
+			rs := n.(*RangeStmt)
+			pruneIfNeeded(rs.Key, n)
+			pruneIfNeeded(rs.Value, n)
+		case OCLOSURE:
+			n := n.(*ClosureExpr)
+			Any(n.Func, do)
+		}
+		return false
+	}
+	Any(fn, do)
+}
+
+// StaticValue method has the same semantics as the ir package function
+// of the same name; see comments on [StaticValue].
+func (ro *ReassignOracle) StaticValue(n Node) Node {
+	arg := n
+	for {
+		if n.Op() == OCONVNOP {
+			n = n.(*ConvExpr).X
+			continue
+		}
+
+		if n.Op() == OINLCALL {
+			n = n.(*InlinedCallExpr).SingleResult()
+			continue
+		}
+
+		n1 := ro.staticValue1(n)
+		if n1 == nil {
+			if consistencyCheckEnabled {
+				checkStaticValueResult(arg, n)
+			}
+			return n
+		}
+		n = n1
+	}
+}
+
+func (ro *ReassignOracle) staticValue1(nn Node) Node {
+	if nn.Op() != ONAME {
+		return nil
+	}
+	n := nn.(*Name).Canonical()
+	if n.Class != PAUTO {
+		return nil
+	}
+
+	defn := n.Defn
+	if defn == nil {
+		return nil
+	}
+
+	var rhs Node
+FindRHS:
+	switch defn.Op() {
+	case OAS:
+		defn := defn.(*AssignStmt)
+		rhs = defn.Y
+	case OAS2:
+		defn := defn.(*AssignListStmt)
+		for i, lhs := range defn.Lhs {
+			if lhs == n {
+				rhs = defn.Rhs[i]
+				break FindRHS
+			}
+		}
+		base.Fatalf("%v missing from LHS of %v", n, defn)
+	default:
+		return nil
+	}
+	if rhs == nil {
+		base.Fatalf("RHS is nil: %v", defn)
+	}
+
+	if _, ok := ro.singleDef[n]; !ok {
+		return nil
+	}
+
+	return rhs
+}
+
+// Reassigned method has the same semantics as the ir package function
+// of the same name; see comments on [Reassigned] for more info.
+func (ro *ReassignOracle) Reassigned(n *Name) bool {
+	_, ok := ro.singleDef[n]
+	result := !ok
+	if consistencyCheckEnabled {
+		checkReassignedResult(n, result)
+	}
+	return result
+}

src/cmd/compile/internal/loong64/galign.go

@@ -20,4 +20,6 @@ func Init(arch *ssagen.ArchInfo) {
 	arch.SSAMarkMoves = func(s *ssagen.State, b *ssa.Block) {}
 	arch.SSAGenValue = ssaGenValue
 	arch.SSAGenBlock = ssaGenBlock
+	arch.LoadRegResult = loadRegResult
+	arch.SpillArgReg = spillArgReg
 }

src/cmd/compile/internal/loong64/ssa.go

@@ -10,6 +10,7 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/logopt"
+	"cmd/compile/internal/objw"
 	"cmd/compile/internal/ssa"
 	"cmd/compile/internal/ssagen"
 	"cmd/compile/internal/types"
@@ -144,6 +145,18 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.From.Type = obj.TYPE_REG
 		p.From.Reg = r
 		ssagen.AddrAuto(&p.To, v)
+	case ssa.OpArgIntReg, ssa.OpArgFloatReg:
+		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
+		// The loop only runs once.
+		for _, a := range v.Block.Func.RegArgs {
+			// Pass the spill/unspill information along to the assembler, offset by size of
+			// the saved LR slot.
+			addr := ssagen.SpillSlotAddr(a, loong64.REGSP, base.Ctxt.Arch.FixedFrameSize)
+			s.FuncInfo().AddSpill(
+				obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type, a.Reg), Spill: storeByType(a.Type, a.Reg)})
+		}
+		v.Block.Func.RegArgs = nil
+		ssagen.CheckArgReg(v)
 	case ssa.OpLOONG64ADDV,
 		ssa.OpLOONG64SUBV,
 		ssa.OpLOONG64AND,
@@ -362,13 +375,12 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = v.Reg()
 	case ssa.OpLOONG64DUFFZERO:
-		// runtime.duffzero expects start address in R19
+		// runtime.duffzero expects start address in R20
 		p := s.Prog(obj.ADUFFZERO)
 		p.To.Type = obj.TYPE_MEM
 		p.To.Name = obj.NAME_EXTERN
 		p.To.Sym = ir.Syms.Duffzero
 		p.To.Offset = v.AuxInt
-
 	case ssa.OpLOONG64LoweredZero:
 		// MOVx	R0, (Rarg0)
 		// ADDV	$sz, Rarg0
@@ -797,3 +809,22 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
 		b.Fatalf("branch not implemented: %s", b.LongString())
 	}
 }
+
+func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
+	p := s.Prog(loadByType(t, reg))
+	p.From.Type = obj.TYPE_MEM
+	p.From.Name = obj.NAME_AUTO
+	p.From.Sym = n.Linksym()
+	p.From.Offset = n.FrameOffset() + off
+	p.To.Type = obj.TYPE_REG
+	p.To.Reg = reg
+	return p
+}
+
+func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
+	p = pp.Append(p, storeByType(t, reg), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
+	p.To.Name = obj.NAME_PARAM
+	p.To.Sym = n.Linksym()
+	p.Pos = p.Pos.WithNotStmt()
+	return p
+}

src/cmd/compile/internal/noder/codes.go

@@ -62,6 +62,7 @@ const (
 	exprFuncInst
 	exprRecv
 	exprReshape
+	exprRuntimeBuiltin // a reference to a runtime function from transformed syntax. Followed by string name, e.g., "panicrangeexit"
 )
 
 type codeAssign int

src/cmd/compile/internal/noder/decl.go

@@ -1,18 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package noder
-
-import (
-	"cmd/compile/internal/syntax"
-	"cmd/compile/internal/types2"
-)
-
-// pkgNameOf returns the PkgName associated with the given ImportDecl.
-func pkgNameOf(info *types2.Info, decl *syntax.ImportDecl) *types2.PkgName {
-	if name := decl.LocalPkgName; name != nil {
-		return info.Defs[name].(*types2.PkgName)
-	}
-	return info.Implicits[decl].(*types2.PkgName)
-}

src/cmd/compile/internal/noder/irgen.go

@@ -61,7 +61,7 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info) {
 		Implicits:          make(map[syntax.Node]types2.Object),
 		Scopes:             make(map[syntax.Node]*types2.Scope),
 		Instances:          make(map[*syntax.Name]types2.Instance),
-		FileVersions:       make(map[*syntax.PosBase]types2.Version),
+		FileVersions:       make(map[*syntax.PosBase]string),
 		// expand as needed
 	}
 	conf.Error = func(err error) {
@@ -72,8 +72,7 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info) {
 			for !posBase.IsFileBase() { // line directive base
 				posBase = posBase.Pos().Base()
 			}
-			v := info.FileVersions[posBase]
-			fileVersion := fmt.Sprintf("go%d.%d", v.Major, v.Minor)
+			fileVersion := info.FileVersions[posBase]
 			file := posBaseMap[posBase]
 			if file.GoVersion == fileVersion {
 				// If we have a version error caused by //go:build, report it.

src/cmd/compile/internal/noder/linker.go

@@ -6,7 +6,6 @@ package noder
 
 import (
 	"internal/buildcfg"
-	"internal/goexperiment"
 	"internal/pkgbits"
 	"io"
 
@@ -297,7 +296,7 @@ func (l *linker) relocFuncExt(w *pkgbits.Encoder, name *ir.Name) {
 	if inl := name.Func.Inl; w.Bool(inl != nil) {
 		w.Len(int(inl.Cost))
 		w.Bool(inl.CanDelayResults)
-		if goexperiment.NewInliner {
+		if buildcfg.Experiment.NewInliner {
 			w.String(inl.Properties)
 		}
 	}

src/cmd/compile/internal/noder/reader.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"go/constant"
 	"internal/buildcfg"
-	"internal/goexperiment"
 	"internal/pkgbits"
 	"path/filepath"
 	"strings"
@@ -16,6 +15,7 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/dwarfgen"
 	"cmd/compile/internal/inline"
+	"cmd/compile/internal/inline/interleaved"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/objw"
 	"cmd/compile/internal/reflectdata"
@@ -1103,7 +1103,7 @@ func (r *reader) funcExt(name *ir.Name, method *types.Sym) {
 				Cost:            int32(r.Len()),
 				CanDelayResults: r.Bool(),
 			}
-			if goexperiment.NewInliner {
+			if buildcfg.Experiment.NewInliner {
 				fn.Inl.Properties = r.String()
 			}
 		}
@@ -2442,6 +2442,10 @@ func (r *reader) expr() (res ir.Node) {
 			n.SetTypecheck(1)
 		}
 		return n
+
+	case exprRuntimeBuiltin:
+		builtin := typecheck.LookupRuntime(r.String())
+		return builtin
 	}
 }
 
@@ -3791,7 +3795,7 @@ func finishWrapperFunc(fn *ir.Func, target *ir.Package) {
 	// We generate wrappers after the global inlining pass,
 	// so we're responsible for applying inlining ourselves here.
 	// TODO(prattmic): plumb PGO.
-	inline.InlineCalls(fn, nil)
+	interleaved.DevirtualizeAndInlineFunc(fn, nil)
 
 	// The body of wrapper function after inlining may reveal new ir.OMETHVALUE node,
 	// we don't know whether wrapper function has been generated for it or not, so

src/cmd/compile/internal/noder/unified.go

@@ -30,8 +30,8 @@ var localPkgReader *pkgReader
 // LookupMethodFunc returns the ir.Func for an arbitrary full symbol name if
 // that function exists in the set of available export data.
 //
-// This allows lookup of arbitrary methods that aren't otherwise referenced by
-// the local package and thus haven't been read yet.
+// This allows lookup of arbitrary functions and methods that aren't otherwise
+// referenced by the local package and thus haven't been read yet.
 //
 // TODO(prattmic): Does not handle instantiation of generic types. Currently
 // profiles don't contain the original type arguments, so we won't be able to
@@ -40,7 +40,7 @@ var localPkgReader *pkgReader
 // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
 // are only used when debug logging is enabled. Consider constructing cheaper
 // errors by default.
-func LookupMethodFunc(fullName string) (*ir.Func, error) {
+func LookupFunc(fullName string) (*ir.Func, error) {
 	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
@@ -51,6 +51,43 @@ func LookupMethodFunc(fullName string) (*ir.Func, error) {
 		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
 	}
 
+	// Symbol naming is ambiguous. We can't necessarily distinguish between
+	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
+	// function Bar, or a method on type Bar? Thus we must simply attempt
+	// to lookup both.
+
+	fn, err := lookupFunction(pkg, symName)
+	if err == nil {
+		return fn, nil
+	}
+
+	fn, mErr := lookupMethod(pkg, symName)
+	if mErr == nil {
+		return fn, nil
+	}
+
+	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
+}
+
+func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
+	sym := pkg.Lookup(symName)
+
+	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
+	// present in objReader, only as OCLOSURE nodes in the enclosing
+	// function.
+	pri, ok := objReader[sym]
+	if !ok {
+		return nil, fmt.Errorf("func sym %v missing objReader", sym)
+	}
+
+	name := pri.pr.objIdx(pri.idx, nil, nil, false).(*ir.Name)
+	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
+		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
+	}
+	return name.Func, nil
+}
+
+func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
 	// N.B. readPackage creates a Sym for every object in the package to
 	// initialize objReader and importBodyReader, even if the object isn't
 	// read.
@@ -130,7 +167,7 @@ func LookupMethodFunc(fullName string) (*ir.Func, error) {
 func unified(m posMap, noders []*noder) {
 	inline.InlineCall = unifiedInlineCall
 	typecheck.HaveInlineBody = unifiedHaveInlineBody
-	pgo.LookupMethodFunc = LookupMethodFunc
+	pgo.LookupFunc = LookupFunc
 
 	data := writePkgStub(m, noders)
 
@@ -243,7 +280,7 @@ func readBodies(target *ir.Package, duringInlining bool) {
 
 		oldLowerM := base.Flag.LowerM
 		base.Flag.LowerM = 0
-		inline.InlineDecls(nil, inlDecls, false)
+		inline.CanInlineFuncs(inlDecls, nil)
 		base.Flag.LowerM = oldLowerM
 
 		for _, fn := range inlDecls {

src/cmd/compile/internal/noder/writer.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"go/constant"
 	"go/token"
+	"go/version"
 	"internal/buildcfg"
 	"internal/pkgbits"
 	"os"
@@ -1479,8 +1480,8 @@ func (w *writer) forStmt(stmt *syntax.ForStmt) {
 
 func (w *writer) distinctVars(stmt *syntax.ForStmt) bool {
 	lv := base.Debug.LoopVar
-	v := w.p.info.FileVersions[stmt.Pos().Base()]
-	is122 := v.Major == 0 && v.Minor == 0 || v.Major == 1 && v.Minor >= 22
+	fileVersion := w.p.info.FileVersions[stmt.Pos().Base()]
+	is122 := fileVersion == "" || version.Compare(fileVersion, "go1.22") >= 0
 
 	// Turning off loopvar for 1.22 is only possible with loopvarhash=qn
 	//
@@ -1714,6 +1715,15 @@ func (w *writer) expr(expr syntax.Expr) {
 	targs := inst.TypeArgs
 
 	if tv, ok := w.p.maybeTypeAndValue(expr); ok {
+		if tv.IsRuntimeHelper() {
+			if pkg := obj.Pkg(); pkg != nil && pkg.Name() == "runtime" {
+				objName := obj.Name()
+				w.Code(exprRuntimeBuiltin)
+				w.String(objName)
+				return
+			}
+		}
+
 		if tv.IsType() {
 			w.p.fatalf(expr, "unexpected type expression %v", syntax.String(expr))
 		}
@@ -2483,7 +2493,7 @@ func (c *declCollector) Visit(n syntax.Node) syntax.Visitor {
 	case *syntax.ImportDecl:
 		pw.checkPragmas(n.Pragma, 0, false)
 
-		switch pkgNameOf(pw.info, n).Imported().Path() {
+		switch pw.info.PkgNameOf(n).Imported().Path() {
 		case "embed":
 			c.file.importedEmbed = true
 		case "unsafe":

src/cmd/compile/internal/pgo/irgraph.go

@@ -366,9 +366,9 @@ func addIREdge(callerNode *IRNode, callerName string, call ir.Node, callee *ir.F
 	callerNode.OutEdges[namedEdge] = edge
 }
 
-// LookupMethodFunc looks up a method in export data. It is expected to be
-// overridden by package noder, to break a dependency cycle.
-var LookupMethodFunc = func(fullName string) (*ir.Func, error) {
+// LookupFunc looks up a function or method in export data. It is expected to
+// be overridden by package noder, to break a dependency cycle.
+var LookupFunc = func(fullName string) (*ir.Func, error) {
 	base.Fatalf("pgo.LookupMethodFunc not overridden")
 	panic("unreachable")
 }
@@ -425,9 +425,7 @@ func addIndirectEdges(g *IRGraph, namedEdgeMap NamedEdgeMap) {
 			// function may still be available from export data of
 			// a transitive dependency.
 			//
-			// TODO(prattmic): Currently we only attempt to lookup
-			// methods because we can only devirtualize interface
-			// calls, not any function pointer. Generic types are
+			// TODO(prattmic): Parameterized types/functions are
 			// not supported.
 			//
 			// TODO(prattmic): This eager lookup during graph load
@@ -437,7 +435,7 @@ func addIndirectEdges(g *IRGraph, namedEdgeMap NamedEdgeMap) {
 			// devirtualization. Instantiation of generic functions
 			// will likely need to be done at the devirtualization
 			// site, if at all.
-			fn, err := LookupMethodFunc(key.CalleeName)
+			fn, err := LookupFunc(key.CalleeName)
 			if err == nil {
 				if base.Debug.PGODebug >= 3 {
 					fmt.Printf("addIndirectEdges: %s found in export data\n", key.CalleeName)

src/cmd/compile/internal/ppc64/ssa.go

@@ -593,7 +593,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = r
 
-	case ssa.OpPPC64ANDCC, ssa.OpPPC64ORCC, ssa.OpPPC64XORCC:
+	case ssa.OpPPC64ADDCC, ssa.OpPPC64ANDCC, ssa.OpPPC64SUBCC, ssa.OpPPC64ORCC, ssa.OpPPC64XORCC, ssa.OpPPC64NORCC,
+		ssa.OpPPC64ANDNCC:
 		r1 := v.Args[0].Reg()
 		r2 := v.Args[1].Reg()
 		p := s.Prog(v.Op.Asm())
@@ -603,6 +604,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = v.Reg0()
 
+	case ssa.OpPPC64NEGCC, ssa.OpPPC64CNTLZDCC:
+		p := s.Prog(v.Op.Asm())
+		p.To.Type = obj.TYPE_REG
+		p.To.Reg = v.Reg0()
+		p.From.Type = obj.TYPE_REG
+		p.From.Reg = v.Args[0].Reg()
+
 	case ssa.OpPPC64ROTLconst, ssa.OpPPC64ROTLWconst:
 		p := s.Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_CONST
@@ -734,13 +742,12 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = v.Reg()
 
-	case ssa.OpPPC64ANDCCconst:
+	case ssa.OpPPC64ADDCCconst, ssa.OpPPC64ANDCCconst:
 		p := s.Prog(v.Op.Asm())
 		p.Reg = v.Args[0].Reg()
 		p.From.Type = obj.TYPE_CONST
 		p.From.Offset = v.AuxInt
 		p.To.Type = obj.TYPE_REG
-		//		p.To.Reg = ppc64.REGTMP // discard result
 		p.To.Reg = v.Reg0()
 
 	case ssa.OpPPC64MOVDaddr:

src/cmd/compile/internal/rangefunc/rewrite.go

@@ -141,6 +141,37 @@ TODO: What about:
 With this rewrite the "return true" is not visible after yield returns,
 but maybe it should be?
 
+# Checking
+
+To permit checking that an iterator is well-behaved -- that is, that
+it does not call the loop body again after it has returned false or
+after the entire loop has exited (it might retain a copy of the body
+function, or pass it to another goroutine) -- each generated loop has
+its own #exitK flag that is checked before each iteration, and set both
+at any early exit and after the iteration completes.
+
+For example:
+
+	for x := range f {
+		...
+		if ... { break }
+		...
+	}
+
+becomes
+
+	{
+		var #exit1 bool
+		f(func(x T1) bool {
+			if #exit1 { runtime.panicrangeexit() }
+			...
+			if ... { #exit1 = true ; return false }
+			...
+			return true
+		})
+		#exit1 = true
+	}
+
 # Nested Loops
 
 So far we've only considered a single loop. If a function contains a
@@ -175,23 +206,30 @@ becomes
 			#r1 type1
 			#r2 type2
 		)
+		var #exit1 bool
 		f(func() {
+			if #exit1 { runtime.panicrangeexit() }
+			var #exit2 bool
 			g(func() {
+				if #exit2 { runtime.panicrangeexit() }
 				...
 				{
 					// return a, b
 					#r1, #r2 = a, b
 					#next = -2
+					#exit1, #exit2 = true, true
 					return false
 				}
 				...
 				return true
 			})
+			#exit2 = true
 			if #next < 0 {
 				return false
 			}
 			return true
 		})
+		#exit1 = true
 		if #next == -2 {
 			return #r1, #r2
 		}
@@ -205,7 +243,8 @@ return with a single check.
 For a labeled break or continue of an outer range-over-func, we
 use positive #next values. Any such labeled break or continue
 really means "do N breaks" or "do N breaks and 1 continue".
-We encode that as 2*N or 2*N+1 respectively.
+We encode that as perLoopStep*N or perLoopStep*N+1 respectively.
+
 Loops that might need to propagate a labeled break or continue
 add one or both of these to the #next checks:
 
@@ -239,30 +278,40 @@ becomes
 
 	{
 		var #next int
+		var #exit1 bool
 		f(func() {
+			if #exit1 { runtime.panicrangeexit() }
+			var #exit2 bool
 			g(func() {
+				if #exit2 { runtime.panicrangeexit() }
+				var #exit3 bool
 				h(func() {
+					if #exit3 { runtime.panicrangeexit() }
 					...
 					{
 						// break F
 						#next = 4
+						#exit1, #exit2, #exit3 = true, true, true
 						return false
 					}
 					...
 					{
 						// continue F
 						#next = 3
+						#exit2, #exit3 = true, true
 						return false
 					}
 					...
 					return true
 				})
+				#exit3 = true
 				if #next >= 2 {
 					#next -= 2
 					return false
 				}
 				return true
 			})
+			#exit2 = true
 			if #next >= 2 {
 				#next -= 2
 				return false
@@ -274,6 +323,7 @@ becomes
 			...
 			return true
 		})
+		#exit1 = true
 	}
 
 Note that the post-h checks only consider a break,
@@ -299,6 +349,7 @@ For example
 	Top: print("start\n")
 	for range f {
 		for range g {
+			...
 			for range h {
 				...
 				goto Top
@@ -312,28 +363,39 @@ becomes
 	Top: print("start\n")
 	{
 		var #next int
+		var #exit1 bool
 		f(func() {
+			if #exit1 { runtime.panicrangeexit() }
+			var #exit2 bool
 			g(func() {
+				if #exit2 { runtime.panicrangeexit() }
+				...
+				var #exit3 bool
 				h(func() {
+				if #exit3 { runtime.panicrangeexit() }
 					...
 					{
 						// goto Top
 						#next = -3
+						#exit1, #exit2, #exit3 = true, true, true
 						return false
 					}
 					...
 					return true
 				})
+				#exit3 = true
 				if #next < 0 {
 					return false
 				}
 				return true
 			})
+			#exit2 = true
 			if #next < 0 {
 				return false
 			}
 			return true
 		})
+		#exit1 = true
 		if #next == -3 {
 			#next = 0
 			goto Top
@@ -431,10 +493,11 @@ type rewriter struct {
 	rewritten map[*syntax.ForStmt]syntax.Stmt
 
 	// Declared variables in generated code for outermost loop.
-	declStmt *syntax.DeclStmt
-	nextVar  types2.Object
-	retVars  []types2.Object
-	defers   types2.Object
+	declStmt     *syntax.DeclStmt
+	nextVar      types2.Object
+	retVars      []types2.Object
+	defers       types2.Object
+	exitVarCount int // exitvars are referenced from their respective loops
 }
 
 // A branch is a single labeled branch.
@@ -445,7 +508,9 @@ type branch struct {
 
 // A forLoop describes a single range-over-func loop being processed.
 type forLoop struct {
-	nfor *syntax.ForStmt // actual syntax
+	nfor         *syntax.ForStmt // actual syntax
+	exitFlag     *types2.Var     // #exit variable for this loop
+	exitFlagDecl *syntax.VarDecl
 
 	checkRet      bool     // add check for "return" after loop
 	checkRetArgs  bool     // add check for "return args" after loop
@@ -489,6 +554,11 @@ func rewriteFunc(pkg *types2.Package, info *types2.Info, typ *syntax.FuncType, b
 	}
 }
 
+// checkFuncMisuse reports whether to check for misuse of iterator callbacks functions.
+func (r *rewriter) checkFuncMisuse() bool {
+	return base.Debug.RangeFuncCheck != 0
+}
+
 // inspect is a callback for syntax.Inspect that drives the actual rewriting.
 // If it sees a func literal, it kicks off a separate rewrite for that literal.
 // Otherwise, it maintains a stack of range-over-func loops and
@@ -556,6 +626,10 @@ func (r *rewriter) startLoop(loop *forLoop) {
 		r.false = types2.Universe.Lookup("false")
 		r.rewritten = make(map[*syntax.ForStmt]syntax.Stmt)
 	}
+	if r.checkFuncMisuse() {
+		// declare the exit flag for this loop's body
+		loop.exitFlag, loop.exitFlagDecl = r.exitVar(loop.nfor.Pos())
+	}
 }
 
 // editStmt returns the replacement for the statement x,
@@ -605,6 +679,19 @@ func (r *rewriter) editDefer(x *syntax.CallStmt) syntax.Stmt {
 	return x
 }
 
+func (r *rewriter) exitVar(pos syntax.Pos) (*types2.Var, *syntax.VarDecl) {
+	r.exitVarCount++
+
+	name := fmt.Sprintf("#exit%d", r.exitVarCount)
+	typ := r.bool.Type()
+	obj := types2.NewVar(pos, r.pkg, name, typ)
+	n := syntax.NewName(pos, name)
+	setValueType(n, typ)
+	r.info.Defs[n] = obj
+
+	return obj, &syntax.VarDecl{NameList: []*syntax.Name{n}}
+}
+
 // editReturn returns the replacement for the return statement x.
 // See the "Return" section in the package doc comment above for more context.
 func (r *rewriter) editReturn(x *syntax.ReturnStmt) syntax.Stmt {
@@ -635,11 +722,24 @@ func (r *rewriter) editReturn(x *syntax.ReturnStmt) syntax.Stmt {
 		bl.List = append(bl.List, &syntax.AssignStmt{Lhs: r.useList(r.retVars), Rhs: x.Results})
 	}
 	bl.List = append(bl.List, &syntax.AssignStmt{Lhs: r.next(), Rhs: r.intConst(next)})
+	if r.checkFuncMisuse() {
+		// mark all enclosing loop bodies as exited
+		for i := 0; i < len(r.forStack); i++ {
+			bl.List = append(bl.List, r.setExitedAt(i))
+		}
+	}
 	bl.List = append(bl.List, &syntax.ReturnStmt{Results: r.useVar(r.false)})
 	setPos(bl, x.Pos())
 	return bl
 }
 
+// perLoopStep is part of the encoding of loop-spanning control flow
+// for function range iterators.  Each multiple of two encodes a "return false"
+// passing control to an enclosing iterator; a terminal value of 1 encodes
+// "return true" (i.e., local continue) from the body function, and a terminal
+// value of 0 encodes executing the remainder of the body function.
+const perLoopStep = 2
+
 // editBranch returns the replacement for the branch statement x,
 // or x itself if it should be left alone.
 // See the package doc comment above for more context.
@@ -660,6 +760,9 @@ func (r *rewriter) editBranch(x *syntax.BranchStmt) syntax.Stmt {
 	for i >= 0 && r.forStack[i].nfor != targ {
 		i--
 	}
+	// exitFrom is the index of the loop interior to the target of the control flow,
+	// if such a loop exists (it does not if i == len(r.forStack) - 1)
+	exitFrom := i + 1
 
 	// Compute the value to assign to #next and the specific return to use.
 	var next int
@@ -688,6 +791,7 @@ func (r *rewriter) editBranch(x *syntax.BranchStmt) syntax.Stmt {
 		for i >= 0 && r.forStack[i].nfor != targ {
 			i--
 		}
+		exitFrom = i + 1
 
 		// Mark loop we exit to get to targ to check for that branch.
 		// When i==-1 that's the outermost func body
@@ -707,34 +811,47 @@ func (r *rewriter) editBranch(x *syntax.BranchStmt) syntax.Stmt {
 
 		// For continue of innermost loop, use "return true".
 		// Otherwise we are breaking the innermost loop, so "return false".
-		retVal := r.false
-		if depth == 0 && x.Tok == syntax.Continue {
-			retVal = r.true
-		}
-		ret = &syntax.ReturnStmt{Results: r.useVar(retVal)}
 
-		// If we're only operating on the innermost loop, the return is all we need.
-		if depth == 0 {
+		if depth == 0 && x.Tok == syntax.Continue {
+			ret = &syntax.ReturnStmt{Results: r.useVar(r.true)}
 			setPos(ret, x.Pos())
 			return ret
 		}
+		ret = &syntax.ReturnStmt{Results: r.useVar(r.false)}
+
+		// If this is a simple break, mark this loop as exited and return false.
+		// No adjustments to #next.
+		if depth == 0 {
+			var stmts []syntax.Stmt
+			if r.checkFuncMisuse() {
+				stmts = []syntax.Stmt{r.setExited(), ret}
+			} else {
+				stmts = []syntax.Stmt{ret}
+			}
+			bl := &syntax.BlockStmt{
+				List: stmts,
+			}
+			setPos(bl, x.Pos())
+			return bl
+		}
 
 		// The loop inside the one we are break/continue-ing
 		// needs to make that happen when we break out of it.
 		if x.Tok == syntax.Continue {
-			r.forStack[i+1].checkContinue = true
+			r.forStack[exitFrom].checkContinue = true
 		} else {
-			r.forStack[i+1].checkBreak = true
+			exitFrom = i
+			r.forStack[exitFrom].checkBreak = true
 		}
 
 		// The loops along the way just need to break.
-		for j := i + 2; j < len(r.forStack); j++ {
+		for j := exitFrom + 1; j < len(r.forStack); j++ {
 			r.forStack[j].checkBreak = true
 		}
 
 		// Set next to break the appropriate number of times;
 		// the final time may be a continue, not a break.
-		next = 2 * depth
+		next = perLoopStep * depth
 		if x.Tok == syntax.Continue {
 			next--
 		}
@@ -743,8 +860,17 @@ func (r *rewriter) editBranch(x *syntax.BranchStmt) syntax.Stmt {
 	// Assign #next = next and do the return.
 	as := &syntax.AssignStmt{Lhs: r.next(), Rhs: r.intConst(next)}
 	bl := &syntax.BlockStmt{
-		List: []syntax.Stmt{as, ret},
+		List: []syntax.Stmt{as},
 	}
+
+	if r.checkFuncMisuse() {
+		// Set #exitK for this loop and those exited by the control flow.
+		for i := exitFrom; i < len(r.forStack); i++ {
+			bl.List = append(bl.List, r.setExitedAt(i))
+		}
+	}
+
+	bl.List = append(bl.List, ret)
 	setPos(bl, x.Pos())
 	return bl
 }
@@ -844,7 +970,20 @@ func (r *rewriter) endLoop(loop *forLoop) {
 		setPos(r.declStmt, start)
 		block.List = append(block.List, r.declStmt)
 	}
+
+	// declare the exitFlag here so it has proper scope and zeroing
+	if r.checkFuncMisuse() {
+		exitFlagDecl := &syntax.DeclStmt{DeclList: []syntax.Decl{loop.exitFlagDecl}}
+		block.List = append(block.List, exitFlagDecl)
+	}
+
+	// iteratorFunc(bodyFunc)
 	block.List = append(block.List, call)
+
+	if r.checkFuncMisuse() {
+		// iteratorFunc has exited, mark the exit flag for the body
+		block.List = append(block.List, r.setExited())
+	}
 	block.List = append(block.List, checks...)
 
 	if len(r.forStack) == 1 { // ending an outermost loop
@@ -857,6 +996,18 @@ func (r *rewriter) endLoop(loop *forLoop) {
 	r.rewritten[nfor] = block
 }
 
+func (r *rewriter) setExited() *syntax.AssignStmt {
+	return r.setExitedAt(len(r.forStack) - 1)
+}
+
+func (r *rewriter) setExitedAt(index int) *syntax.AssignStmt {
+	loop := r.forStack[index]
+	return &syntax.AssignStmt{
+		Lhs: r.useVar(loop.exitFlag),
+		Rhs: r.useVar(r.true),
+	}
+}
+
 // bodyFunc converts the loop body (control flow has already been updated)
 // to a func literal that can be passed to the range function.
 //
@@ -909,6 +1060,12 @@ func (r *rewriter) bodyFunc(body []syntax.Stmt, lhs []syntax.Expr, def bool, fty
 	tv.SetIsValue()
 	bodyFunc.SetTypeInfo(tv)
 
+	loop := r.forStack[len(r.forStack)-1]
+
+	if r.checkFuncMisuse() {
+		bodyFunc.Body.List = append(bodyFunc.Body.List, r.assertNotExited(start, loop))
+	}
+
 	// Original loop body (already rewritten by editStmt during inspect).
 	bodyFunc.Body.List = append(bodyFunc.Body.List, body...)
 
@@ -948,10 +1105,10 @@ func (r *rewriter) checks(loop *forLoop, pos syntax.Pos) []syntax.Stmt {
 			list = append(list, r.ifNext(syntax.Lss, 0, retStmt(r.useVar(r.false))))
 		}
 		if loop.checkBreak {
-			list = append(list, r.ifNext(syntax.Geq, 2, retStmt(r.useVar(r.false))))
+			list = append(list, r.ifNext(syntax.Geq, perLoopStep, retStmt(r.useVar(r.false))))
 		}
 		if loop.checkContinue {
-			list = append(list, r.ifNext(syntax.Eql, 1, retStmt(r.useVar(r.true))))
+			list = append(list, r.ifNext(syntax.Eql, perLoopStep-1, retStmt(r.useVar(r.true))))
 		}
 	}
 
@@ -1003,6 +1160,36 @@ func (r *rewriter) ifNext(op syntax.Operator, c int, then syntax.Stmt) syntax.St
 	return nif
 }
 
+// setValueType marks x as a value with type typ.
+func setValueType(x syntax.Expr, typ syntax.Type) {
+	tv := syntax.TypeAndValue{Type: typ}
+	tv.SetIsValue()
+	x.SetTypeInfo(tv)
+}
+
+// assertNotExited returns the statement:
+//
+//	if #exitK { runtime.panicrangeexit() }
+//
+// where #exitK is the exit guard for loop.
+func (r *rewriter) assertNotExited(start syntax.Pos, loop *forLoop) syntax.Stmt {
+	callPanicExpr := &syntax.CallExpr{
+		Fun: runtimeSym(r.info, "panicrangeexit"),
+	}
+	setValueType(callPanicExpr, nil) // no result type
+
+	callPanic := &syntax.ExprStmt{X: callPanicExpr}
+
+	nif := &syntax.IfStmt{
+		Cond: r.useVar(loop.exitFlag),
+		Then: &syntax.BlockStmt{
+			List: []syntax.Stmt{callPanic},
+		},
+	}
+	setPos(nif, start)
+	return nif
+}
+
 // next returns a reference to the #next variable.
 func (r *rewriter) next() *syntax.Name {
 	if r.nextVar == nil {
@@ -1099,7 +1286,11 @@ var runtimePkg = func() *types2.Package {
 	anyType := types2.Universe.Lookup("any").Type()
 
 	// func deferrangefunc() unsafe.Pointer
-	obj := types2.NewVar(nopos, pkg, "deferrangefunc", types2.NewSignatureType(nil, nil, nil, nil, types2.NewTuple(types2.NewParam(nopos, pkg, "extra", anyType)), false))
+	obj := types2.NewFunc(nopos, pkg, "deferrangefunc", types2.NewSignatureType(nil, nil, nil, nil, types2.NewTuple(types2.NewParam(nopos, pkg, "extra", anyType)), false))
+	pkg.Scope().Insert(obj)
+
+	// func panicrangeexit()
+	obj = types2.NewFunc(nopos, pkg, "panicrangeexit", types2.NewSignatureType(nil, nil, nil, nil, nil, false))
 	pkg.Scope().Insert(obj)
 
 	return pkg
@@ -1111,6 +1302,7 @@ func runtimeSym(info *types2.Info, name string) *syntax.Name {
 	n := syntax.NewName(nopos, "runtime."+name)
 	tv := syntax.TypeAndValue{Type: obj.Type()}
 	tv.SetIsValue()
+	tv.SetIsRuntimeHelper()
 	n.SetTypeInfo(tv)
 	info.Uses[n] = obj
 	return n

src/cmd/compile/internal/ssa/_gen/ARM.rules

@@ -66,17 +66,17 @@
 
 // count trailing zero for ARMv5 and ARMv6
 // 32 - CLZ(x&-x - 1)
-(Ctz32 <t> x) && buildcfg.GOARM<=6 =>
+(Ctz32 <t> x) && buildcfg.GOARM.Version<=6 =>
 	(RSBconst [32] (CLZ <t> (SUBconst <t> (AND <t> x (RSBconst <t> [0] x)) [1])))
-(Ctz16 <t> x) && buildcfg.GOARM<=6 =>
+(Ctz16 <t> x) && buildcfg.GOARM.Version<=6 =>
 	(RSBconst [32] (CLZ <t> (SUBconst <typ.UInt32> (AND <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x) (RSBconst <typ.UInt32> [0] (ORconst <typ.UInt32> [0x10000] x))) [1])))
-(Ctz8 <t> x) && buildcfg.GOARM<=6 =>
+(Ctz8 <t> x) && buildcfg.GOARM.Version<=6 =>
 	(RSBconst [32] (CLZ <t> (SUBconst <typ.UInt32> (AND <typ.UInt32> (ORconst <typ.UInt32> [0x100] x) (RSBconst <typ.UInt32> [0] (ORconst <typ.UInt32> [0x100] x))) [1])))
 
 // count trailing zero for ARMv7
-(Ctz32 <t> x) && buildcfg.GOARM==7 => (CLZ <t> (RBIT <t> x))
-(Ctz16 <t> x) && buildcfg.GOARM==7 => (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x)))
-(Ctz8 <t> x) && buildcfg.GOARM==7 => (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x100] x)))
+(Ctz32 <t> x) && buildcfg.GOARM.Version==7 => (CLZ <t> (RBIT <t> x))
+(Ctz16 <t> x) && buildcfg.GOARM.Version==7 => (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x)))
+(Ctz8 <t> x) && buildcfg.GOARM.Version==7 => (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x100] x)))
 
 // bit length
 (BitLen32 <t> x) => (RSBconst [32] (CLZ <t> x))
@@ -90,13 +90,13 @@
 // t5 = x right rotate 8 bits  -- (d,   a,   b,   c  )
 // result = t4 ^ t5            -- (d,   c,   b,   a  )
 // using shifted ops this can be done in 4 instructions.
-(Bswap32 <t> x) && buildcfg.GOARM==5 =>
+(Bswap32 <t> x) && buildcfg.GOARM.Version==5 =>
 	(XOR <t>
 		(SRLconst <t> (BICconst <t> (XOR <t> x (SRRconst <t> [16] x)) [0xff0000]) [8])
 		(SRRconst <t> x [8]))
 
 // byte swap for ARMv6 and above
-(Bswap32 x) && buildcfg.GOARM>=6 => (REV x)
+(Bswap32 x) && buildcfg.GOARM.Version>=6 => (REV x)
 
 // boolean ops -- booleans are represented with 0=false, 1=true
 (AndB ...) => (AND ...)
@@ -741,10 +741,10 @@
 (SUBconst [c] x) && !isARMImmRot(uint32(c)) && isARMImmRot(uint32(-c)) => (ADDconst [-c] x)
 (ANDconst [c] x) && !isARMImmRot(uint32(c)) && isARMImmRot(^uint32(c)) => (BICconst [int32(^uint32(c))] x)
 (BICconst [c] x) && !isARMImmRot(uint32(c)) && isARMImmRot(^uint32(c)) => (ANDconst [int32(^uint32(c))] x)
-(ADDconst [c] x) && buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff => (SUBconst [-c] x)
-(SUBconst [c] x) && buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff => (ADDconst [-c] x)
-(ANDconst [c] x) && buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff => (BICconst [int32(^uint32(c))] x)
-(BICconst [c] x) && buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff => (ANDconst [int32(^uint32(c))] x)
+(ADDconst [c] x) && buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff => (SUBconst [-c] x)
+(SUBconst [c] x) && buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff => (ADDconst [-c] x)
+(ANDconst [c] x) && buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff => (BICconst [int32(^uint32(c))] x)
+(BICconst [c] x) && buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff => (ANDconst [int32(^uint32(c))] x)
 (ADDconst [c] (MOVWconst [d])) => (MOVWconst [c+d])
 (ADDconst [c] (ADDconst [d] x)) => (ADDconst [c+d] x)
 (ADDconst [c] (SUBconst [d] x)) => (ADDconst [c-d] x)
@@ -1139,7 +1139,7 @@
 // UBFX instruction is supported by ARMv6T2, ARMv7 and above versions, REV16 is supported by
 // ARMv6 and above versions. So for ARMv6, we need to match SLLconst, SRLconst and ORshiftLL.
 ((ADDshiftLL|ORshiftLL|XORshiftLL) <typ.UInt16> [8] (BFXU <typ.UInt16> [int32(armBFAuxInt(8, 8))] x) x) => (REV16 x)
-((ADDshiftLL|ORshiftLL|XORshiftLL) <typ.UInt16> [8] (SRLconst <typ.UInt16> [24] (SLLconst [16] x)) x) && buildcfg.GOARM>=6 => (REV16 x)
+((ADDshiftLL|ORshiftLL|XORshiftLL) <typ.UInt16> [8] (SRLconst <typ.UInt16> [24] (SLLconst [16] x)) x) && buildcfg.GOARM.Version>=6 => (REV16 x)
 
 // use indexed loads and stores
 (MOVWload [0] {sym} (ADD ptr idx) mem) && sym == nil => (MOVWloadidx ptr idx mem)
@@ -1209,25 +1209,25 @@
 (BIC x x) => (MOVWconst [0])
 
 (ADD (MUL x y) a) => (MULA x y a)
-(SUB a (MUL x y)) && buildcfg.GOARM == 7 => (MULS x y a)
-(RSB (MUL x y) a) && buildcfg.GOARM == 7 => (MULS x y a)
+(SUB a (MUL x y)) && buildcfg.GOARM.Version == 7 => (MULS x y a)
+(RSB (MUL x y) a) && buildcfg.GOARM.Version == 7 => (MULS x y a)
 
-(NEGF (MULF x y)) && buildcfg.GOARM >= 6 => (NMULF x y)
-(NEGD (MULD x y)) && buildcfg.GOARM >= 6 => (NMULD x y)
-(MULF (NEGF x) y) && buildcfg.GOARM >= 6 => (NMULF x y)
-(MULD (NEGD x) y) && buildcfg.GOARM >= 6 => (NMULD x y)
+(NEGF (MULF x y)) && buildcfg.GOARM.Version >= 6 => (NMULF x y)
+(NEGD (MULD x y)) && buildcfg.GOARM.Version >= 6 => (NMULD x y)
+(MULF (NEGF x) y) && buildcfg.GOARM.Version >= 6 => (NMULF x y)
+(MULD (NEGD x) y) && buildcfg.GOARM.Version >= 6 => (NMULD x y)
 (NMULF (NEGF x) y) => (MULF x y)
 (NMULD (NEGD x) y) => (MULD x y)
 
 // the result will overwrite the addend, since they are in the same register
-(ADDF a (MULF x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULAF a x y)
-(ADDF a (NMULF x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULSF a x y)
-(ADDD a (MULD x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULAD a x y)
-(ADDD a (NMULD x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULSD a x y)
-(SUBF a (MULF x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULSF a x y)
-(SUBF a (NMULF x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULAF a x y)
-(SUBD a (MULD x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULSD a x y)
-(SUBD a (NMULD x y)) && a.Uses == 1 && buildcfg.GOARM >= 6 => (MULAD a x y)
+(ADDF a (MULF x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULAF a x y)
+(ADDF a (NMULF x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULSF a x y)
+(ADDD a (MULD x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULAD a x y)
+(ADDD a (NMULD x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULSD a x y)
+(SUBF a (MULF x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULSF a x y)
+(SUBF a (NMULF x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULAF a x y)
+(SUBD a (MULD x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULSD a x y)
+(SUBD a (NMULD x y)) && a.Uses == 1 && buildcfg.GOARM.Version >= 6 => (MULAD a x y)
 
 (AND x (MVN y)) => (BIC x y)
 
@@ -1259,8 +1259,8 @@
 (CMPD x (MOVDconst [0])) => (CMPD0 x)
 
 // bit extraction
-(SRAconst (SLLconst x [c]) [d]) && buildcfg.GOARM==7 && uint64(d)>=uint64(c) && uint64(d)<=31 => (BFX [(d-c)|(32-d)<<8] x)
-(SRLconst (SLLconst x [c]) [d]) && buildcfg.GOARM==7 && uint64(d)>=uint64(c) && uint64(d)<=31 => (BFXU [(d-c)|(32-d)<<8] x)
+(SRAconst (SLLconst x [c]) [d]) && buildcfg.GOARM.Version==7 && uint64(d)>=uint64(c) && uint64(d)<=31 => (BFX [(d-c)|(32-d)<<8] x)
+(SRLconst (SLLconst x [c]) [d]) && buildcfg.GOARM.Version==7 && uint64(d)>=uint64(c) && uint64(d)<=31 => (BFXU [(d-c)|(32-d)<<8] x)
 
 // comparison simplification
 ((EQ|NE) (CMP x (RSBconst [0] y))) => ((EQ|NE) (CMN x y)) // sense of carry bit not preserved; see also #50854

src/cmd/compile/internal/ssa/_gen/LOONG64.rules

@@ -416,7 +416,7 @@
 (GetCallerSP ...) => (LoweredGetCallerSP ...)
 (GetCallerPC ...) => (LoweredGetCallerPC ...)
 
-(If cond yes no) => (NE cond yes no)
+(If cond yes no) => (NE (MOVBUreg <typ.UInt64> cond) yes no)
 
 // Write barrier.
 (WB ...) => (LoweredWB ...)
@@ -450,71 +450,37 @@
 (EQ (SGTconst [0] x) yes no) => (GEZ x yes no)
 (NE (SGT x (MOVVconst [0])) yes no) => (GTZ x yes no)
 (EQ (SGT x (MOVVconst [0])) yes no) => (LEZ x yes no)
+(MOVBUreg x:((SGT|SGTU) _ _)) => x
 
 // fold offset into address
 (ADDVconst [off1] (MOVVaddr [off2] {sym} ptr)) && is32Bit(off1+int64(off2)) => (MOVVaddr [int32(off1)+int32(off2)] {sym} ptr)
 
 // fold address into load/store
-(MOVBload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVBload  [off1+int32(off2)] {sym} ptr mem)
-(MOVBUload [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVBUload [off1+int32(off2)] {sym} ptr mem)
-(MOVHload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVHload  [off1+int32(off2)] {sym} ptr mem)
-(MOVHUload [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVHUload [off1+int32(off2)] {sym} ptr mem)
-(MOVWload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVWload  [off1+int32(off2)] {sym} ptr mem)
-(MOVWUload [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVWUload [off1+int32(off2)] {sym} ptr mem)
-(MOVVload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVVload  [off1+int32(off2)] {sym} ptr mem)
-(MOVFload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVFload  [off1+int32(off2)] {sym} ptr mem)
-(MOVDload  [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVDload  [off1+int32(off2)] {sym} ptr mem)
+// Do not fold global variable access in -dynlink mode, where it will be rewritten
+// to use the GOT via REGTMP, which currently cannot handle large offset.
+(MOV(B|BU|H|HU|W|WU|V|F|D)load [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2)
+	&& (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|BU|H|HU|W|WU|V|F|D)load [off1+int32(off2)] {sym} ptr mem)
 
-(MOVBstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVBstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVHstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVHstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVWstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVWstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVVstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVVstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVFstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVFstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVDstore [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2) => (MOVDstore [off1+int32(off2)] {sym} ptr val mem)
-(MOVBstorezero [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVBstorezero [off1+int32(off2)] {sym} ptr mem)
-(MOVHstorezero [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVHstorezero [off1+int32(off2)] {sym} ptr mem)
-(MOVWstorezero [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVWstorezero [off1+int32(off2)] {sym} ptr mem)
-(MOVVstorezero [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2) => (MOVVstorezero [off1+int32(off2)] {sym} ptr mem)
+(MOV(B|H|W|V|F|D)store [off1] {sym} (ADDVconst [off2] ptr) val mem) && is32Bit(int64(off1)+off2)
+	&& (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|H|W|V|F|D)store [off1+int32(off2)] {sym} ptr val mem)
 
-(MOVBload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVBload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVBUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVBUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVHload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVHload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVHUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVHUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVWload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVWload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVWUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVWUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVVload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVVload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVFload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVFload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVDload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVDload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
+(MOV(B|H|W|V)storezero [off1] {sym} (ADDVconst [off2] ptr) mem) && is32Bit(int64(off1)+off2)
+	&& (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|H|W|V)storezero [off1+int32(off2)] {sym} ptr mem)
 
-(MOVBstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVBstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVHstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVHstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVWstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVWstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVVstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVVstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVFstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVFstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVDstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVDstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
-(MOVBstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVBstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVHstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVHstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVWstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVWstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
-(MOVVstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) =>
-	(MOVVstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
+(MOV(B|BU|H|HU|W|WU|V|F|D)load [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2)
+	&& is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|BU|H|HU|W|WU|V|F|D)load [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
+
+(MOV(B|H|W|V|F|D)store [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem) && canMergeSym(sym1,sym2)
+	&& is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|H|W|V|F|D)store [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
+
+(MOV(B|H|W|V)storezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem) && canMergeSym(sym1,sym2)
+	&& is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink) =>
+	(MOV(B|H|W|V)storezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 
 (LoweredAtomicStore(32|64) ptr (MOVVconst [0]) mem) => (LoweredAtomicStorezero(32|64) ptr mem)
 (LoweredAtomicAdd32 ptr (MOVVconst [c]) mem) && is32Bit(c) => (LoweredAtomicAddconst32 [int32(c)] ptr mem)

src/cmd/compile/internal/ssa/_gen/LOONG64Ops.go

@@ -123,17 +123,17 @@ func init() {
 
 	// Common individual register masks
 	var (
-		gp         = buildReg("R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R23 R24 R25 R26 R27 R28 R29 R31") // R1 is LR, R2 is thread pointer, R3 is stack pointer, R21-unused, R22 is g, R30 is REGTMP
+		gp         = buildReg("R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R23 R24 R25 R26 R27 R28 R29 R31") // R1 is LR, R2 is thread pointer, R3 is stack pointer, R22 is g, R30 is REGTMP
 		gpg        = gp | buildReg("g")
 		gpsp       = gp | buildReg("SP")
 		gpspg      = gpg | buildReg("SP")
 		gpspsbg    = gpspg | buildReg("SB")
 		fp         = buildReg("F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31")
 		callerSave = gp | fp | buildReg("g") // runtime.setg (and anything calling it) may clobber g
-		r1         = buildReg("R19")
-		r2         = buildReg("R18")
-		r3         = buildReg("R17")
-		r4         = buildReg("R4")
+		r1         = buildReg("R20")
+		r2         = buildReg("R21")
+		r3         = buildReg("R23")
+		r4         = buildReg("R24")
 	)
 	// Common regInfo
 	var (
@@ -273,32 +273,32 @@ func init() {
 		{name: "MOVDF", argLength: 1, reg: fp11, asm: "MOVDF"},     // float64 -> float32
 
 		// function calls
-		{name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                               // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
-		{name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true},                                 // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
-		{name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("R29"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
-		{name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
+		{name: "CALLstatic", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                               // call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
+		{name: "CALLtail", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true},                                 // tail call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
+		{name: "CALLclosure", argLength: -1, reg: regInfo{inputs: []regMask{gpsp, buildReg("R29"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, last arg=mem, auxint=argsize, returns mem
+		{name: "CALLinter", argLength: -1, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, last arg=mem, auxint=argsize, returns mem
 
 		// duffzero
 		// arg0 = address of memory to zero
 		// arg1 = mem
 		// auxint = offset into duffzero code to start executing
 		// returns mem
-		// R19 aka loong64.REGRT1 changed as side effect
+		// R20 aka loong64.REGRT1 changed as side effect
 		{
 			name:      "DUFFZERO",
 			aux:       "Int64",
 			argLength: 2,
 			reg: regInfo{
-				inputs:   []regMask{buildReg("R19")},
-				clobbers: buildReg("R19 R1"),
+				inputs:   []regMask{buildReg("R20")},
+				clobbers: buildReg("R20 R1"),
 			},
 			typ:            "Mem",
 			faultOnNilArg0: true,
 		},
 
 		// duffcopy
-		// arg0 = address of dst memory (in R20, changed as side effect) REGRT2
-		// arg1 = address of src memory (in R19, changed as side effect) REGRT1
+		// arg0 = address of dst memory (in R21, changed as side effect)
+		// arg1 = address of src memory (in R20, changed as side effect)
 		// arg2 = mem
 		// auxint = offset into duffcopy code to start executing
 		// returns mem
@@ -307,8 +307,8 @@ func init() {
 			aux:       "Int64",
 			argLength: 3,
 			reg: regInfo{
-				inputs:   []regMask{buildReg("R20"), buildReg("R19")},
-				clobbers: buildReg("R19 R20 R1"),
+				inputs:   []regMask{buildReg("R21"), buildReg("R20")},
+				clobbers: buildReg("R20 R21 R1"),
 			},
 			typ:            "Mem",
 			faultOnNilArg0: true,
@@ -316,45 +316,45 @@ func init() {
 		},
 
 		// large or unaligned zeroing
-		// arg0 = address of memory to zero (in R19, changed as side effect)
+		// arg0 = address of memory to zero (in R20, changed as side effect)
 		// arg1 = address of the last element to zero
 		// arg2 = mem
 		// auxint = alignment
 		// returns mem
-		//	MOVx	R0, (R19)
-		//	ADDV	$sz, R19
-		//	BGEU	Rarg1, R19, -2(PC)
+		//	MOVx	R0, (R20)
+		//	ADDV	$sz, R20
+		//	BGEU	Rarg1, R20, -2(PC)
 		{
 			name:      "LoweredZero",
 			aux:       "Int64",
 			argLength: 3,
 			reg: regInfo{
-				inputs:   []regMask{buildReg("R19"), gp},
-				clobbers: buildReg("R19"),
+				inputs:   []regMask{buildReg("R20"), gp},
+				clobbers: buildReg("R20"),
 			},
 			typ:            "Mem",
 			faultOnNilArg0: true,
 		},
 
 		// large or unaligned move
-		// arg0 = address of dst memory (in R20, changed as side effect)
-		// arg1 = address of src memory (in R19, changed as side effect)
+		// arg0 = address of dst memory (in R21, changed as side effect)
+		// arg1 = address of src memory (in R20, changed as side effect)
 		// arg2 = address of the last element of src
 		// arg3 = mem
 		// auxint = alignment
 		// returns mem
-		//	MOVx	(R19), Rtmp
-		//	MOVx	Rtmp, (R20)
-		//	ADDV	$sz, R19
+		//	MOVx	(R20), Rtmp
+		//	MOVx	Rtmp, (R21)
 		//	ADDV	$sz, R20
-		//	BGEU	Rarg2, R19, -4(PC)
+		//	ADDV	$sz, R21
+		//	BGEU	Rarg2, R20, -4(PC)
 		{
 			name:      "LoweredMove",
 			aux:       "Int64",
 			argLength: 4,
 			reg: regInfo{
-				inputs:   []regMask{buildReg("R20"), buildReg("R19"), gp},
-				clobbers: buildReg("R19 R20"),
+				inputs:   []regMask{buildReg("R21"), buildReg("R20"), gp},
+				clobbers: buildReg("R20 R21"),
 			},
 			typ:            "Mem",
 			faultOnNilArg0: true,
@@ -476,8 +476,8 @@ func init() {
 		blocks:   blocks,
 		regnames: regNamesLOONG64,
 		// TODO: support register ABI on loong64
-		ParamIntRegNames:   "R4 R5 R6 R7 R8 R9 R10 R11",
-		ParamFloatRegNames: "F0 F1 F2 F3 F4 F5 F6 F7",
+		ParamIntRegNames:   "R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19",
+		ParamFloatRegNames: "F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15",
 		gpregmask:          gp,
 		fpregmask:          fp,
 		framepointerreg:    -1, // not used

src/cmd/compile/internal/ssa/_gen/PPC64Ops.go

@@ -176,14 +176,17 @@ func init() {
 		r6          = buildReg("R6")
 	)
 	ops := []opData{
-		{name: "ADD", argLength: 2, reg: gp21, asm: "ADD", commutative: true},        // arg0 + arg1
-		{name: "ADDconst", argLength: 1, reg: gp11, asm: "ADD", aux: "Int64"},        // arg0 + auxInt
-		{name: "FADD", argLength: 2, reg: fp21, asm: "FADD", commutative: true},      // arg0+arg1
-		{name: "FADDS", argLength: 2, reg: fp21, asm: "FADDS", commutative: true},    // arg0+arg1
-		{name: "SUB", argLength: 2, reg: gp21, asm: "SUB"},                           // arg0-arg1
-		{name: "SUBFCconst", argLength: 1, reg: gp11cxer, asm: "SUBC", aux: "Int64"}, // auxInt - arg0 (carry is ignored)
-		{name: "FSUB", argLength: 2, reg: fp21, asm: "FSUB"},                         // arg0-arg1
-		{name: "FSUBS", argLength: 2, reg: fp21, asm: "FSUBS"},                       // arg0-arg1
+		{name: "ADD", argLength: 2, reg: gp21, asm: "ADD", commutative: true},                              // arg0 + arg1
+		{name: "ADDCC", argLength: 2, reg: gp21, asm: "ADDCC", commutative: true, typ: "(Int,Flags)"},      // arg0 + arg1
+		{name: "ADDconst", argLength: 1, reg: gp11, asm: "ADD", aux: "Int64"},                              // arg0 + auxInt
+		{name: "ADDCCconst", argLength: 1, reg: gp11cxer, asm: "ADDCCC", aux: "Int64", typ: "(Int,Flags)"}, // arg0 + auxInt sets CC, clobbers XER
+		{name: "FADD", argLength: 2, reg: fp21, asm: "FADD", commutative: true},                            // arg0+arg1
+		{name: "FADDS", argLength: 2, reg: fp21, asm: "FADDS", commutative: true},                          // arg0+arg1
+		{name: "SUB", argLength: 2, reg: gp21, asm: "SUB"},                                                 // arg0-arg1
+		{name: "SUBCC", argLength: 2, reg: gp21, asm: "SUBCC", typ: "(Int,Flags)"},                         // arg0-arg1 sets CC
+		{name: "SUBFCconst", argLength: 1, reg: gp11cxer, asm: "SUBC", aux: "Int64"},                       // auxInt - arg0 (carry is ignored)
+		{name: "FSUB", argLength: 2, reg: fp21, asm: "FSUB"},                                               // arg0-arg1
+		{name: "FSUBS", argLength: 2, reg: fp21, asm: "FSUBS"},                                             // arg0-arg1
 
 		{name: "MULLD", argLength: 2, reg: gp21, asm: "MULLD", typ: "Int64", commutative: true}, // arg0*arg1 (signed 64-bit)
 		{name: "MULLW", argLength: 2, reg: gp21, asm: "MULLW", typ: "Int32", commutative: true}, // arg0*arg1 (signed 32-bit)
@@ -245,8 +248,9 @@ func init() {
 		{name: "RLDICL", argLength: 1, reg: gp11, asm: "RLDICL", aux: "Int64"},                     // Auxint is encoded similarly to RLWINM, but only MB and SH are valid. ME is always 63.
 		{name: "RLDICR", argLength: 1, reg: gp11, asm: "RLDICR", aux: "Int64"},                     // Likewise, but only ME and SH are valid. MB is always 0.
 
-		{name: "CNTLZD", argLength: 1, reg: gp11, asm: "CNTLZD", clobberFlags: true}, // count leading zeros
-		{name: "CNTLZW", argLength: 1, reg: gp11, asm: "CNTLZW", clobberFlags: true}, // count leading zeros (32 bit)
+		{name: "CNTLZD", argLength: 1, reg: gp11, asm: "CNTLZD"},                          // count leading zeros
+		{name: "CNTLZDCC", argLength: 1, reg: gp11, asm: "CNTLZDCC", typ: "(Int, Flags)"}, // count leading zeros, sets CC
+		{name: "CNTLZW", argLength: 1, reg: gp11, asm: "CNTLZW"},                          // count leading zeros (32 bit)
 
 		{name: "CNTTZD", argLength: 1, reg: gp11, asm: "CNTTZD"}, // count trailing zeros
 		{name: "CNTTZW", argLength: 1, reg: gp11, asm: "CNTTZW"}, // count trailing zeros (32 bit)
@@ -285,34 +289,37 @@ func init() {
 		{name: "MFVSRD", argLength: 1, reg: fpgp, asm: "MFVSRD", typ: "Int64"},   // move 64 bits of F register into G register
 		{name: "MTVSRD", argLength: 1, reg: gpfp, asm: "MTVSRD", typ: "Float64"}, // move 64 bits of G register into F register
 
-		{name: "AND", argLength: 2, reg: gp21, asm: "AND", commutative: true},                                               // arg0&arg1
-		{name: "ANDN", argLength: 2, reg: gp21, asm: "ANDN"},                                                                // arg0&^arg1
-		{name: "ANDCC", argLength: 2, reg: gp21, asm: "ANDCC", commutative: true, clobberFlags: true, typ: "(Int64,Flags)"}, // arg0&arg1 sets CC
-		{name: "OR", argLength: 2, reg: gp21, asm: "OR", commutative: true},                                                 // arg0|arg1
-		{name: "ORN", argLength: 2, reg: gp21, asm: "ORN"},                                                                  // arg0|^arg1
-		{name: "ORCC", argLength: 2, reg: gp21, asm: "ORCC", commutative: true, clobberFlags: true, typ: "(Int,Flags)"},     // arg0|arg1 sets CC
-		{name: "NOR", argLength: 2, reg: gp21, asm: "NOR", commutative: true},                                               // ^(arg0|arg1)
-		{name: "XOR", argLength: 2, reg: gp21, asm: "XOR", typ: "Int64", commutative: true},                                 // arg0^arg1
-		{name: "XORCC", argLength: 2, reg: gp21, asm: "XORCC", commutative: true, clobberFlags: true, typ: "(Int,Flags)"},   // arg0^arg1 sets CC
-		{name: "EQV", argLength: 2, reg: gp21, asm: "EQV", typ: "Int64", commutative: true},                                 // arg0^^arg1
-		{name: "NEG", argLength: 1, reg: gp11, asm: "NEG"},                                                                  // -arg0 (integer)
-		{name: "BRD", argLength: 1, reg: gp11, asm: "BRD"},                                                                  // reversebytes64(arg0)
-		{name: "BRW", argLength: 1, reg: gp11, asm: "BRW"},                                                                  // reversebytes32(arg0)
-		{name: "BRH", argLength: 1, reg: gp11, asm: "BRH"},                                                                  // reversebytes16(arg0)
-		{name: "FNEG", argLength: 1, reg: fp11, asm: "FNEG"},                                                                // -arg0 (floating point)
-		{name: "FSQRT", argLength: 1, reg: fp11, asm: "FSQRT"},                                                              // sqrt(arg0) (floating point)
-		{name: "FSQRTS", argLength: 1, reg: fp11, asm: "FSQRTS"},                                                            // sqrt(arg0) (floating point, single precision)
-		{name: "FFLOOR", argLength: 1, reg: fp11, asm: "FRIM"},                                                              // floor(arg0), float64
-		{name: "FCEIL", argLength: 1, reg: fp11, asm: "FRIP"},                                                               // ceil(arg0), float64
-		{name: "FTRUNC", argLength: 1, reg: fp11, asm: "FRIZ"},                                                              // trunc(arg0), float64
-		{name: "FROUND", argLength: 1, reg: fp11, asm: "FRIN"},                                                              // round(arg0), float64
-		{name: "FABS", argLength: 1, reg: fp11, asm: "FABS"},                                                                // abs(arg0), float64
-		{name: "FNABS", argLength: 1, reg: fp11, asm: "FNABS"},                                                              // -abs(arg0), float64
-		{name: "FCPSGN", argLength: 2, reg: fp21, asm: "FCPSGN"},                                                            // copysign arg0 -> arg1, float64
+		{name: "AND", argLength: 2, reg: gp21, asm: "AND", commutative: true},                           // arg0&arg1
+		{name: "ANDN", argLength: 2, reg: gp21, asm: "ANDN"},                                            // arg0&^arg1
+		{name: "ANDNCC", argLength: 2, reg: gp21, asm: "ANDNCC", typ: "(Int64,Flags)"},                  // arg0&^arg1 sets CC
+		{name: "ANDCC", argLength: 2, reg: gp21, asm: "ANDCC", commutative: true, typ: "(Int64,Flags)"}, // arg0&arg1 sets CC
+		{name: "OR", argLength: 2, reg: gp21, asm: "OR", commutative: true},                             // arg0|arg1
+		{name: "ORN", argLength: 2, reg: gp21, asm: "ORN"},                                              // arg0|^arg1
+		{name: "ORCC", argLength: 2, reg: gp21, asm: "ORCC", commutative: true, typ: "(Int,Flags)"},     // arg0|arg1 sets CC
+		{name: "NOR", argLength: 2, reg: gp21, asm: "NOR", commutative: true},                           // ^(arg0|arg1)
+		{name: "NORCC", argLength: 2, reg: gp21, asm: "NORCC", commutative: true, typ: "(Int,Flags)"},   // ^(arg0|arg1) sets CC
+		{name: "XOR", argLength: 2, reg: gp21, asm: "XOR", typ: "Int64", commutative: true},             // arg0^arg1
+		{name: "XORCC", argLength: 2, reg: gp21, asm: "XORCC", commutative: true, typ: "(Int,Flags)"},   // arg0^arg1 sets CC
+		{name: "EQV", argLength: 2, reg: gp21, asm: "EQV", typ: "Int64", commutative: true},             // arg0^^arg1
+		{name: "NEG", argLength: 1, reg: gp11, asm: "NEG"},                                              // -arg0 (integer)
+		{name: "NEGCC", argLength: 1, reg: gp11, asm: "NEGCC", typ: "(Int,Flags)"},                      // -arg0 (integer) sets CC
+		{name: "BRD", argLength: 1, reg: gp11, asm: "BRD"},                                              // reversebytes64(arg0)
+		{name: "BRW", argLength: 1, reg: gp11, asm: "BRW"},                                              // reversebytes32(arg0)
+		{name: "BRH", argLength: 1, reg: gp11, asm: "BRH"},                                              // reversebytes16(arg0)
+		{name: "FNEG", argLength: 1, reg: fp11, asm: "FNEG"},                                            // -arg0 (floating point)
+		{name: "FSQRT", argLength: 1, reg: fp11, asm: "FSQRT"},                                          // sqrt(arg0) (floating point)
+		{name: "FSQRTS", argLength: 1, reg: fp11, asm: "FSQRTS"},                                        // sqrt(arg0) (floating point, single precision)
+		{name: "FFLOOR", argLength: 1, reg: fp11, asm: "FRIM"},                                          // floor(arg0), float64
+		{name: "FCEIL", argLength: 1, reg: fp11, asm: "FRIP"},                                           // ceil(arg0), float64
+		{name: "FTRUNC", argLength: 1, reg: fp11, asm: "FRIZ"},                                          // trunc(arg0), float64
+		{name: "FROUND", argLength: 1, reg: fp11, asm: "FRIN"},                                          // round(arg0), float64
+		{name: "FABS", argLength: 1, reg: fp11, asm: "FABS"},                                            // abs(arg0), float64
+		{name: "FNABS", argLength: 1, reg: fp11, asm: "FNABS"},                                          // -abs(arg0), float64
+		{name: "FCPSGN", argLength: 2, reg: fp21, asm: "FCPSGN"},                                        // copysign arg0 -> arg1, float64
 
-		{name: "ORconst", argLength: 1, reg: gp11, asm: "OR", aux: "Int64"},   // arg0|aux
-		{name: "XORconst", argLength: 1, reg: gp11, asm: "XOR", aux: "Int64"}, // arg0^aux
-		{name: "ANDCCconst", argLength: 1, reg: regInfo{inputs: []regMask{gp | sp | sb}, outputs: []regMask{gp}}, asm: "ANDCC", aux: "Int64", clobberFlags: true, typ: "(Int,Flags)"}, // arg0&aux == 0 // and-immediate sets CC on PPC, always.
+		{name: "ORconst", argLength: 1, reg: gp11, asm: "OR", aux: "Int64"},                                                                                       // arg0|aux
+		{name: "XORconst", argLength: 1, reg: gp11, asm: "XOR", aux: "Int64"},                                                                                     // arg0^aux
+		{name: "ANDCCconst", argLength: 1, reg: regInfo{inputs: []regMask{gp | sp | sb}, outputs: []regMask{gp}}, asm: "ANDCC", aux: "Int64", typ: "(Int,Flags)"}, // arg0&aux == 0 // and-immediate sets CC on PPC, always.
 
 		{name: "MOVBreg", argLength: 1, reg: gp11, asm: "MOVB", typ: "Int64"},   // sign extend int8 to int64
 		{name: "MOVBZreg", argLength: 1, reg: gp11, asm: "MOVBZ", typ: "Int64"}, // zero extend uint8 to uint64

src/cmd/compile/internal/ssa/_gen/PPC64latelower.rules

@@ -36,3 +36,20 @@
 
 // When PCRel is supported, paddi can add a 34b signed constant in one instruction.
 (ADD (MOVDconst [m]) x) && supportsPPC64PCRel() && (m<<30)>>30 == m => (ADDconst [m] x)
+
+
+// Where possible and practical, generate CC opcodes. Due to the structure of the rules, there are limits to how
+// a Value can be rewritten which make it impossible to correctly rewrite sibling Value users. To workaround this
+// case, candidates for CC opcodes are converted in two steps:
+//   1. Convert all (x (Op ...) ...) into (x (Select0 (OpCC ...) ...). See convertPPC64OpToOpCC for more
+//      detail on how and why this is done there.
+//   2. Rewrite (CMPconst [0] (Select0 (OpCC ...))) into (Select1 (OpCC...))
+// Note: to minimize potentially expensive regeneration of CC opcodes during the flagalloc pass, only rewrite if
+//       both ops are in the same block.
+(CMPconst [0] z:((ADD|AND|ANDN|OR|SUB|NOR|XOR) x y)) && v.Block == z.Block => (CMPconst [0] convertPPC64OpToOpCC(z))
+(CMPconst [0] z:((NEG|CNTLZD) x)) && v.Block == z.Block => (CMPconst [0] convertPPC64OpToOpCC(z))
+// Note: ADDCCconst only assembles to 1 instruction for int16 constants.
+(CMPconst [0] z:(ADDconst [c] x)) && int64(int16(c)) == c && v.Block == z.Block => (CMPconst [0] convertPPC64OpToOpCC(z))
+// And finally, fixup the flag user.
+(CMPconst <t> [0] (Select0 z:((ADD|AND|ANDN|OR|SUB|NOR|XOR)CC x y))) => (Select1 <t> z)
+(CMPconst <t> [0] (Select0 z:((ADDCCconst|NEGCC|CNTLZDCC) y))) => (Select1 <t> z)

src/cmd/compile/internal/ssa/_gen/allocators.go

@@ -1,3 +1,7 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package main
 
 // TODO: should we share backing storage for similarly-shaped types?

src/cmd/compile/internal/ssa/config.go

@@ -283,6 +283,8 @@ func NewConfig(arch string, types Types, ctxt *obj.Link, optimize, softfloat boo
 		c.registers = registersLOONG64[:]
 		c.gpRegMask = gpRegMaskLOONG64
 		c.fpRegMask = fpRegMaskLOONG64
+		c.intParamRegs = paramIntRegLOONG64
+		c.floatParamRegs = paramFloatRegLOONG64
 		c.FPReg = framepointerRegLOONG64
 		c.LinkReg = linkRegLOONG64
 		c.hasGReg = true

src/cmd/compile/internal/ssa/debug.go

@@ -42,7 +42,10 @@ type FuncDebug struct {
 	OptDcl []*ir.Name
 
 	// Filled in by the user. Translates Block and Value ID to PC.
-	GetPC func(ID, ID) int64
+	//
+	// NOTE: block is only used if value is BlockStart.ID or BlockEnd.ID.
+	// Otherwise, it is ignored.
+	GetPC func(block, value ID) int64
 }
 
 type BlockDebug struct {
@@ -1368,7 +1371,7 @@ func (state *debugState) buildLocationLists(blockLocs []*BlockDebug) {
 
 	// Flush any leftover entries live at the end of the last block.
 	for varID := range state.lists {
-		state.writePendingEntry(VarID(varID), state.f.Blocks[len(state.f.Blocks)-1].ID, FuncEnd.ID)
+		state.writePendingEntry(VarID(varID), -1, FuncEnd.ID)
 		list := state.lists[varID]
 		if state.loggingLevel > 0 {
 			if len(list) == 0 {

src/cmd/compile/internal/ssa/debug_lines_test.go

@@ -44,7 +44,7 @@ func testGoArch() string {
 
 func hasRegisterABI() bool {
 	switch testGoArch() {
-	case "amd64", "arm64", "ppc64", "ppc64le", "riscv":
+	case "amd64", "arm64", "loong64", "ppc64", "ppc64le", "riscv":
 		return true
 	}
 	return false

src/cmd/compile/internal/ssa/loopbce.go

@@ -129,6 +129,13 @@ func findIndVar(f *Func) []indVar {
 			less = false
 		}
 
+		if ind.Block != b {
+			// TODO: Could be extended to include disjointed loop headers.
+			// I don't think this is causing missed optimizations in real world code often.
+			// See https://go.dev/issue/63955
+			continue
+		}
+
 		// Expect the increment to be a nonzero constant.
 		if !inc.isGenericIntConst() {
 			continue

src/cmd/compile/internal/ssa/magic.go

@@ -170,7 +170,7 @@ func smagicOK(n uint, c int64) bool {
 	return c&(c-1) != 0
 }
 
-// smagicOKn reports whether we should strength reduce an signed n-bit divide by c.
+// smagicOKn reports whether we should strength reduce a signed n-bit divide by c.
 func smagicOK8(c int8) bool   { return smagicOK(8, int64(c)) }
 func smagicOK16(c int16) bool { return smagicOK(16, int64(c)) }
 func smagicOK32(c int32) bool { return smagicOK(32, int64(c)) }

src/cmd/compile/internal/ssa/nilcheck_test.go

@@ -1,3 +1,7 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package ssa
 
 import (

src/cmd/compile/internal/ssa/regalloc.go

@@ -672,6 +672,8 @@ func (s *regAllocState) init(f *Func) {
 			s.allocatable &^= 1 << 9 // R9
 		case "arm64":
 			// nothing to do
+		case "loong64": // R2 (aka TP) already reserved.
+			// nothing to do
 		case "ppc64le": // R2 already reserved.
 			// nothing to do
 		case "riscv64": // X3 (aka GP) and X4 (aka TP) already reserved.

src/cmd/compile/internal/ssa/rewrite.go

@@ -1630,6 +1630,52 @@ func mergePPC64SldiSrw(sld, srw int64) int64 {
 	return encodePPC64RotateMask((32-srw+sld)&31, int64(mask), 32)
 }
 
+// Convert a PPC64 opcode from the Op to OpCC form. This converts (op x y)
+// to (Select0 (opCC x y)) without having to explicitly fixup every user
+// of op.
+//
+// E.g consider the case:
+// a = (ADD x y)
+// b = (CMPconst [0] a)
+// c = (OR a z)
+//
+// A rule like (CMPconst [0] (ADD x y)) => (CMPconst [0] (Select0 (ADDCC x y)))
+// would produce:
+// a  = (ADD x y)
+// a' = (ADDCC x y)
+// a” = (Select0 a')
+// b  = (CMPconst [0] a”)
+// c  = (OR a z)
+//
+// which makes it impossible to rewrite the second user. Instead the result
+// of this conversion is:
+// a' = (ADDCC x y)
+// a  = (Select0 a')
+// b  = (CMPconst [0] a)
+// c  = (OR a z)
+//
+// Which makes it trivial to rewrite b using a lowering rule.
+func convertPPC64OpToOpCC(op *Value) *Value {
+	ccOpMap := map[Op]Op{
+		OpPPC64ADD:      OpPPC64ADDCC,
+		OpPPC64ADDconst: OpPPC64ADDCCconst,
+		OpPPC64AND:      OpPPC64ANDCC,
+		OpPPC64ANDN:     OpPPC64ANDNCC,
+		OpPPC64CNTLZD:   OpPPC64CNTLZDCC,
+		OpPPC64OR:       OpPPC64ORCC,
+		OpPPC64SUB:      OpPPC64SUBCC,
+		OpPPC64NEG:      OpPPC64NEGCC,
+		OpPPC64NOR:      OpPPC64NORCC,
+		OpPPC64XOR:      OpPPC64XORCC,
+	}
+	b := op.Block
+	opCC := b.NewValue0I(op.Pos, ccOpMap[op.Op], types.NewTuple(op.Type, types.TypeFlags), op.AuxInt)
+	opCC.AddArgs(op.Args...)
+	op.reset(OpSelect0)
+	op.AddArgs(opCC)
+	return op
+}
+
 // Convenience function to rotate a 32 bit constant value by another constant.
 func rotateLeft32(v, rotate int64) int64 {
 	return int64(bits.RotateLeft32(uint32(v), int(rotate)))

src/cmd/compile/internal/ssa/rewriteARM.go

@@ -1496,7 +1496,7 @@ func rewriteValueARM_OpARMADDD(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (ADDD a (MULD x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULAD a x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -1506,7 +1506,7 @@ func rewriteValueARM_OpARMADDD(v *Value) bool {
 			}
 			y := v_1.Args[1]
 			x := v_1.Args[0]
-			if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+			if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMMULAD)
@@ -1516,7 +1516,7 @@ func rewriteValueARM_OpARMADDD(v *Value) bool {
 		break
 	}
 	// match: (ADDD a (NMULD x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULSD a x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -1526,7 +1526,7 @@ func rewriteValueARM_OpARMADDD(v *Value) bool {
 			}
 			y := v_1.Args[1]
 			x := v_1.Args[0]
-			if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+			if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMMULSD)
@@ -1541,7 +1541,7 @@ func rewriteValueARM_OpARMADDF(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (ADDF a (MULF x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULAF a x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -1551,7 +1551,7 @@ func rewriteValueARM_OpARMADDF(v *Value) bool {
 			}
 			y := v_1.Args[1]
 			x := v_1.Args[0]
-			if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+			if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMMULAF)
@@ -1561,7 +1561,7 @@ func rewriteValueARM_OpARMADDF(v *Value) bool {
 		break
 	}
 	// match: (ADDF a (NMULF x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULSF a x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -1571,7 +1571,7 @@ func rewriteValueARM_OpARMADDF(v *Value) bool {
 			}
 			y := v_1.Args[1]
 			x := v_1.Args[0]
-			if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+			if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMMULSF)
@@ -1979,12 +1979,12 @@ func rewriteValueARM_OpARMADDconst(v *Value) bool {
 		return true
 	}
 	// match: (ADDconst [c] x)
-	// cond: buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff
+	// cond: buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff
 	// result: (SUBconst [-c] x)
 	for {
 		c := auxIntToInt32(v.AuxInt)
 		x := v_0
-		if !(buildcfg.GOARM == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && uint32(-c) <= 0xffff) {
+		if !(buildcfg.GOARM.Version == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && uint32(-c) <= 0xffff) {
 			break
 		}
 		v.reset(OpARMSUBconst)
@@ -2099,7 +2099,7 @@ func rewriteValueARM_OpARMADDshiftLL(v *Value) bool {
 		return true
 	}
 	// match: (ADDshiftLL <typ.UInt16> [8] (SRLconst <typ.UInt16> [24] (SLLconst [16] x)) x)
-	// cond: buildcfg.GOARM>=6
+	// cond: buildcfg.GOARM.Version>=6
 	// result: (REV16 x)
 	for {
 		if v.Type != typ.UInt16 || auxIntToInt32(v.AuxInt) != 8 || v_0.Op != OpARMSRLconst || v_0.Type != typ.UInt16 || auxIntToInt32(v_0.AuxInt) != 24 {
@@ -2110,7 +2110,7 @@ func rewriteValueARM_OpARMADDshiftLL(v *Value) bool {
 			break
 		}
 		x := v_0_0.Args[0]
-		if x != v_1 || !(buildcfg.GOARM >= 6) {
+		if x != v_1 || !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMREV16)
@@ -2551,12 +2551,12 @@ func rewriteValueARM_OpARMANDconst(v *Value) bool {
 		return true
 	}
 	// match: (ANDconst [c] x)
-	// cond: buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff
+	// cond: buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff
 	// result: (BICconst [int32(^uint32(c))] x)
 	for {
 		c := auxIntToInt32(v.AuxInt)
 		x := v_0
-		if !(buildcfg.GOARM == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && ^uint32(c) <= 0xffff) {
+		if !(buildcfg.GOARM.Version == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && ^uint32(c) <= 0xffff) {
 			break
 		}
 		v.reset(OpARMBICconst)
@@ -3052,12 +3052,12 @@ func rewriteValueARM_OpARMBICconst(v *Value) bool {
 		return true
 	}
 	// match: (BICconst [c] x)
-	// cond: buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff
+	// cond: buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && ^uint32(c)<=0xffff
 	// result: (ANDconst [int32(^uint32(c))] x)
 	for {
 		c := auxIntToInt32(v.AuxInt)
 		x := v_0
-		if !(buildcfg.GOARM == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && ^uint32(c) <= 0xffff) {
+		if !(buildcfg.GOARM.Version == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && ^uint32(c) <= 0xffff) {
 			break
 		}
 		v.reset(OpARMANDconst)
@@ -7590,7 +7590,7 @@ func rewriteValueARM_OpARMMULD(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (MULD (NEGD x) y)
-	// cond: buildcfg.GOARM >= 6
+	// cond: buildcfg.GOARM.Version >= 6
 	// result: (NMULD x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -7599,7 +7599,7 @@ func rewriteValueARM_OpARMMULD(v *Value) bool {
 			}
 			x := v_0.Args[0]
 			y := v_1
-			if !(buildcfg.GOARM >= 6) {
+			if !(buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMNMULD)
@@ -7614,7 +7614,7 @@ func rewriteValueARM_OpARMMULF(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (MULF (NEGF x) y)
-	// cond: buildcfg.GOARM >= 6
+	// cond: buildcfg.GOARM.Version >= 6
 	// result: (NMULF x y)
 	for {
 		for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 {
@@ -7623,7 +7623,7 @@ func rewriteValueARM_OpARMMULF(v *Value) bool {
 			}
 			x := v_0.Args[0]
 			y := v_1
-			if !(buildcfg.GOARM >= 6) {
+			if !(buildcfg.GOARM.Version >= 6) {
 				continue
 			}
 			v.reset(OpARMNMULF)
@@ -8247,7 +8247,7 @@ func rewriteValueARM_OpARMMVNshiftRLreg(v *Value) bool {
 func rewriteValueARM_OpARMNEGD(v *Value) bool {
 	v_0 := v.Args[0]
 	// match: (NEGD (MULD x y))
-	// cond: buildcfg.GOARM >= 6
+	// cond: buildcfg.GOARM.Version >= 6
 	// result: (NMULD x y)
 	for {
 		if v_0.Op != OpARMMULD {
@@ -8255,7 +8255,7 @@ func rewriteValueARM_OpARMNEGD(v *Value) bool {
 		}
 		y := v_0.Args[1]
 		x := v_0.Args[0]
-		if !(buildcfg.GOARM >= 6) {
+		if !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMNMULD)
@@ -8267,7 +8267,7 @@ func rewriteValueARM_OpARMNEGD(v *Value) bool {
 func rewriteValueARM_OpARMNEGF(v *Value) bool {
 	v_0 := v.Args[0]
 	// match: (NEGF (MULF x y))
-	// cond: buildcfg.GOARM >= 6
+	// cond: buildcfg.GOARM.Version >= 6
 	// result: (NMULF x y)
 	for {
 		if v_0.Op != OpARMMULF {
@@ -8275,7 +8275,7 @@ func rewriteValueARM_OpARMNEGF(v *Value) bool {
 		}
 		y := v_0.Args[1]
 		x := v_0.Args[0]
-		if !(buildcfg.GOARM >= 6) {
+		if !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMNMULF)
@@ -8583,7 +8583,7 @@ func rewriteValueARM_OpARMORshiftLL(v *Value) bool {
 		return true
 	}
 	// match: (ORshiftLL <typ.UInt16> [8] (SRLconst <typ.UInt16> [24] (SLLconst [16] x)) x)
-	// cond: buildcfg.GOARM>=6
+	// cond: buildcfg.GOARM.Version>=6
 	// result: (REV16 x)
 	for {
 		if v.Type != typ.UInt16 || auxIntToInt32(v.AuxInt) != 8 || v_0.Op != OpARMSRLconst || v_0.Type != typ.UInt16 || auxIntToInt32(v_0.AuxInt) != 24 {
@@ -8594,7 +8594,7 @@ func rewriteValueARM_OpARMORshiftLL(v *Value) bool {
 			break
 		}
 		x := v_0_0.Args[0]
-		if x != v_1 || !(buildcfg.GOARM >= 6) {
+		if x != v_1 || !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMREV16)
@@ -9048,7 +9048,7 @@ func rewriteValueARM_OpARMRSB(v *Value) bool {
 		return true
 	}
 	// match: (RSB (MUL x y) a)
-	// cond: buildcfg.GOARM == 7
+	// cond: buildcfg.GOARM.Version == 7
 	// result: (MULS x y a)
 	for {
 		if v_0.Op != OpARMMUL {
@@ -9057,7 +9057,7 @@ func rewriteValueARM_OpARMRSB(v *Value) bool {
 		y := v_0.Args[1]
 		x := v_0.Args[0]
 		a := v_1
-		if !(buildcfg.GOARM == 7) {
+		if !(buildcfg.GOARM.Version == 7) {
 			break
 		}
 		v.reset(OpARMMULS)
@@ -10534,7 +10534,7 @@ func rewriteValueARM_OpARMSRAconst(v *Value) bool {
 		return true
 	}
 	// match: (SRAconst (SLLconst x [c]) [d])
-	// cond: buildcfg.GOARM==7 && uint64(d)>=uint64(c) && uint64(d)<=31
+	// cond: buildcfg.GOARM.Version==7 && uint64(d)>=uint64(c) && uint64(d)<=31
 	// result: (BFX [(d-c)|(32-d)<<8] x)
 	for {
 		d := auxIntToInt32(v.AuxInt)
@@ -10543,7 +10543,7 @@ func rewriteValueARM_OpARMSRAconst(v *Value) bool {
 		}
 		c := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
-		if !(buildcfg.GOARM == 7 && uint64(d) >= uint64(c) && uint64(d) <= 31) {
+		if !(buildcfg.GOARM.Version == 7 && uint64(d) >= uint64(c) && uint64(d) <= 31) {
 			break
 		}
 		v.reset(OpARMBFX)
@@ -10590,7 +10590,7 @@ func rewriteValueARM_OpARMSRLconst(v *Value) bool {
 		return true
 	}
 	// match: (SRLconst (SLLconst x [c]) [d])
-	// cond: buildcfg.GOARM==7 && uint64(d)>=uint64(c) && uint64(d)<=31
+	// cond: buildcfg.GOARM.Version==7 && uint64(d)>=uint64(c) && uint64(d)<=31
 	// result: (BFXU [(d-c)|(32-d)<<8] x)
 	for {
 		d := auxIntToInt32(v.AuxInt)
@@ -10599,7 +10599,7 @@ func rewriteValueARM_OpARMSRLconst(v *Value) bool {
 		}
 		c := auxIntToInt32(v_0.AuxInt)
 		x := v_0.Args[0]
-		if !(buildcfg.GOARM == 7 && uint64(d) >= uint64(c) && uint64(d) <= 31) {
+		if !(buildcfg.GOARM.Version == 7 && uint64(d) >= uint64(c) && uint64(d) <= 31) {
 			break
 		}
 		v.reset(OpARMBFXU)
@@ -10830,7 +10830,7 @@ func rewriteValueARM_OpARMSUB(v *Value) bool {
 		return true
 	}
 	// match: (SUB a (MUL x y))
-	// cond: buildcfg.GOARM == 7
+	// cond: buildcfg.GOARM.Version == 7
 	// result: (MULS x y a)
 	for {
 		a := v_0
@@ -10839,7 +10839,7 @@ func rewriteValueARM_OpARMSUB(v *Value) bool {
 		}
 		y := v_1.Args[1]
 		x := v_1.Args[0]
-		if !(buildcfg.GOARM == 7) {
+		if !(buildcfg.GOARM.Version == 7) {
 			break
 		}
 		v.reset(OpARMMULS)
@@ -10852,7 +10852,7 @@ func rewriteValueARM_OpARMSUBD(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (SUBD a (MULD x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULSD a x y)
 	for {
 		a := v_0
@@ -10861,7 +10861,7 @@ func rewriteValueARM_OpARMSUBD(v *Value) bool {
 		}
 		y := v_1.Args[1]
 		x := v_1.Args[0]
-		if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+		if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMMULSD)
@@ -10869,7 +10869,7 @@ func rewriteValueARM_OpARMSUBD(v *Value) bool {
 		return true
 	}
 	// match: (SUBD a (NMULD x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULAD a x y)
 	for {
 		a := v_0
@@ -10878,7 +10878,7 @@ func rewriteValueARM_OpARMSUBD(v *Value) bool {
 		}
 		y := v_1.Args[1]
 		x := v_1.Args[0]
-		if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+		if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMMULAD)
@@ -10891,7 +10891,7 @@ func rewriteValueARM_OpARMSUBF(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
 	// match: (SUBF a (MULF x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULSF a x y)
 	for {
 		a := v_0
@@ -10900,7 +10900,7 @@ func rewriteValueARM_OpARMSUBF(v *Value) bool {
 		}
 		y := v_1.Args[1]
 		x := v_1.Args[0]
-		if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+		if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMMULSF)
@@ -10908,7 +10908,7 @@ func rewriteValueARM_OpARMSUBF(v *Value) bool {
 		return true
 	}
 	// match: (SUBF a (NMULF x y))
-	// cond: a.Uses == 1 && buildcfg.GOARM >= 6
+	// cond: a.Uses == 1 && buildcfg.GOARM.Version >= 6
 	// result: (MULAF a x y)
 	for {
 		a := v_0
@@ -10917,7 +10917,7 @@ func rewriteValueARM_OpARMSUBF(v *Value) bool {
 		}
 		y := v_1.Args[1]
 		x := v_1.Args[0]
-		if !(a.Uses == 1 && buildcfg.GOARM >= 6) {
+		if !(a.Uses == 1 && buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMMULAF)
@@ -11383,12 +11383,12 @@ func rewriteValueARM_OpARMSUBconst(v *Value) bool {
 		return true
 	}
 	// match: (SUBconst [c] x)
-	// cond: buildcfg.GOARM==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff
+	// cond: buildcfg.GOARM.Version==7 && !isARMImmRot(uint32(c)) && uint32(c)>0xffff && uint32(-c)<=0xffff
 	// result: (ADDconst [-c] x)
 	for {
 		c := auxIntToInt32(v.AuxInt)
 		x := v_0
-		if !(buildcfg.GOARM == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && uint32(-c) <= 0xffff) {
+		if !(buildcfg.GOARM.Version == 7 && !isARMImmRot(uint32(c)) && uint32(c) > 0xffff && uint32(-c) <= 0xffff) {
 			break
 		}
 		v.reset(OpARMADDconst)
@@ -12710,7 +12710,7 @@ func rewriteValueARM_OpARMXORshiftLL(v *Value) bool {
 		return true
 	}
 	// match: (XORshiftLL <typ.UInt16> [8] (SRLconst <typ.UInt16> [24] (SLLconst [16] x)) x)
-	// cond: buildcfg.GOARM>=6
+	// cond: buildcfg.GOARM.Version>=6
 	// result: (REV16 x)
 	for {
 		if v.Type != typ.UInt16 || auxIntToInt32(v.AuxInt) != 8 || v_0.Op != OpARMSRLconst || v_0.Type != typ.UInt16 || auxIntToInt32(v_0.AuxInt) != 24 {
@@ -12721,7 +12721,7 @@ func rewriteValueARM_OpARMXORshiftLL(v *Value) bool {
 			break
 		}
 		x := v_0_0.Args[0]
-		if x != v_1 || !(buildcfg.GOARM >= 6) {
+		if x != v_1 || !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMREV16)
@@ -13062,12 +13062,12 @@ func rewriteValueARM_OpBswap32(v *Value) bool {
 	v_0 := v.Args[0]
 	b := v.Block
 	// match: (Bswap32 <t> x)
-	// cond: buildcfg.GOARM==5
+	// cond: buildcfg.GOARM.Version==5
 	// result: (XOR <t> (SRLconst <t> (BICconst <t> (XOR <t> x (SRRconst <t> [16] x)) [0xff0000]) [8]) (SRRconst <t> x [8]))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM == 5) {
+		if !(buildcfg.GOARM.Version == 5) {
 			break
 		}
 		v.reset(OpARMXOR)
@@ -13090,11 +13090,11 @@ func rewriteValueARM_OpBswap32(v *Value) bool {
 		return true
 	}
 	// match: (Bswap32 x)
-	// cond: buildcfg.GOARM>=6
+	// cond: buildcfg.GOARM.Version>=6
 	// result: (REV x)
 	for {
 		x := v_0
-		if !(buildcfg.GOARM >= 6) {
+		if !(buildcfg.GOARM.Version >= 6) {
 			break
 		}
 		v.reset(OpARMREV)
@@ -13177,12 +13177,12 @@ func rewriteValueARM_OpCtz16(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Ctz16 <t> x)
-	// cond: buildcfg.GOARM<=6
+	// cond: buildcfg.GOARM.Version<=6
 	// result: (RSBconst [32] (CLZ <t> (SUBconst <typ.UInt32> (AND <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x) (RSBconst <typ.UInt32> [0] (ORconst <typ.UInt32> [0x10000] x))) [1])))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM <= 6) {
+		if !(buildcfg.GOARM.Version <= 6) {
 			break
 		}
 		v.reset(OpARMRSBconst)
@@ -13204,12 +13204,12 @@ func rewriteValueARM_OpCtz16(v *Value) bool {
 		return true
 	}
 	// match: (Ctz16 <t> x)
-	// cond: buildcfg.GOARM==7
+	// cond: buildcfg.GOARM.Version==7
 	// result: (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x)))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM == 7) {
+		if !(buildcfg.GOARM.Version == 7) {
 			break
 		}
 		v.reset(OpARMCLZ)
@@ -13228,12 +13228,12 @@ func rewriteValueARM_OpCtz32(v *Value) bool {
 	v_0 := v.Args[0]
 	b := v.Block
 	// match: (Ctz32 <t> x)
-	// cond: buildcfg.GOARM<=6
+	// cond: buildcfg.GOARM.Version<=6
 	// result: (RSBconst [32] (CLZ <t> (SUBconst <t> (AND <t> x (RSBconst <t> [0] x)) [1])))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM <= 6) {
+		if !(buildcfg.GOARM.Version <= 6) {
 			break
 		}
 		v.reset(OpARMRSBconst)
@@ -13252,12 +13252,12 @@ func rewriteValueARM_OpCtz32(v *Value) bool {
 		return true
 	}
 	// match: (Ctz32 <t> x)
-	// cond: buildcfg.GOARM==7
+	// cond: buildcfg.GOARM.Version==7
 	// result: (CLZ <t> (RBIT <t> x))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM == 7) {
+		if !(buildcfg.GOARM.Version == 7) {
 			break
 		}
 		v.reset(OpARMCLZ)
@@ -13274,12 +13274,12 @@ func rewriteValueARM_OpCtz8(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Ctz8 <t> x)
-	// cond: buildcfg.GOARM<=6
+	// cond: buildcfg.GOARM.Version<=6
 	// result: (RSBconst [32] (CLZ <t> (SUBconst <typ.UInt32> (AND <typ.UInt32> (ORconst <typ.UInt32> [0x100] x) (RSBconst <typ.UInt32> [0] (ORconst <typ.UInt32> [0x100] x))) [1])))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM <= 6) {
+		if !(buildcfg.GOARM.Version <= 6) {
 			break
 		}
 		v.reset(OpARMRSBconst)
@@ -13301,12 +13301,12 @@ func rewriteValueARM_OpCtz8(v *Value) bool {
 		return true
 	}
 	// match: (Ctz8 <t> x)
-	// cond: buildcfg.GOARM==7
+	// cond: buildcfg.GOARM.Version==7
 	// result: (CLZ <t> (RBIT <typ.UInt32> (ORconst <typ.UInt32> [0x100] x)))
 	for {
 		t := v.Type
 		x := v_0
-		if !(buildcfg.GOARM == 7) {
+		if !(buildcfg.GOARM.Version == 7) {
 			break
 		}
 		v.reset(OpARMCLZ)

src/cmd/compile/internal/ssa/rewriteLOONG64.go

@@ -1724,8 +1724,10 @@ func rewriteValueLOONG64_OpLOONG64MASKNEZ(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVBUload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVBUload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBUload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1736,7 +1738,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBUload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBUload)
@@ -1746,7 +1748,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBUload(v *Value) bool {
 		return true
 	}
 	// match: (MOVBUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1758,7 +1760,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBUload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBUload)
@@ -1771,6 +1773,26 @@ func rewriteValueLOONG64_OpLOONG64MOVBUload(v *Value) bool {
 }
 func rewriteValueLOONG64_OpLOONG64MOVBUreg(v *Value) bool {
 	v_0 := v.Args[0]
+	// match: (MOVBUreg x:(SGT _ _))
+	// result: x
+	for {
+		x := v_0
+		if x.Op != OpLOONG64SGT {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
+	// match: (MOVBUreg x:(SGTU _ _))
+	// result: x
+	for {
+		x := v_0
+		if x.Op != OpLOONG64SGTU {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
 	// match: (MOVBUreg x:(MOVBUload _ _))
 	// result: (MOVVreg x)
 	for {
@@ -1809,8 +1831,10 @@ func rewriteValueLOONG64_OpLOONG64MOVBUreg(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVBload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVBload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1821,7 +1845,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBload)
@@ -1831,7 +1855,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBload(v *Value) bool {
 		return true
 	}
 	// match: (MOVBload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1843,7 +1867,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBload)
@@ -1895,8 +1919,10 @@ func rewriteValueLOONG64_OpLOONG64MOVBstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVBstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1908,7 +1934,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBstore)
@@ -1918,7 +1944,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVBstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -1931,7 +1957,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBstore)
@@ -2047,8 +2073,10 @@ func rewriteValueLOONG64_OpLOONG64MOVBstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVBstorezero(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVBstorezero [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBstorezero [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2059,7 +2087,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstorezero(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBstorezero)
@@ -2069,7 +2097,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstorezero(v *Value) bool {
 		return true
 	}
 	// match: (MOVBstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVBstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2081,7 +2109,7 @@ func rewriteValueLOONG64_OpLOONG64MOVBstorezero(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVBstorezero)
@@ -2095,8 +2123,10 @@ func rewriteValueLOONG64_OpLOONG64MOVBstorezero(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVDload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVDload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVDload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2107,7 +2137,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVDload)
@@ -2117,7 +2147,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDload(v *Value) bool {
 		return true
 	}
 	// match: (MOVDload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVDload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2129,7 +2159,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVDload)
@@ -2144,8 +2174,10 @@ func rewriteValueLOONG64_OpLOONG64MOVDstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVDstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVDstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2157,7 +2189,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVDstore)
@@ -2167,7 +2199,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVDstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVDstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2180,7 +2212,7 @@ func rewriteValueLOONG64_OpLOONG64MOVDstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVDstore)
@@ -2194,8 +2226,10 @@ func rewriteValueLOONG64_OpLOONG64MOVDstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVFload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVFload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVFload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2206,7 +2240,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVFload)
@@ -2216,7 +2250,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFload(v *Value) bool {
 		return true
 	}
 	// match: (MOVFload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVFload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2228,7 +2262,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVFload)
@@ -2243,8 +2277,10 @@ func rewriteValueLOONG64_OpLOONG64MOVFstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVFstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVFstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2256,7 +2292,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVFstore)
@@ -2266,7 +2302,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVFstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVFstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2279,7 +2315,7 @@ func rewriteValueLOONG64_OpLOONG64MOVFstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVFstore)
@@ -2293,8 +2329,10 @@ func rewriteValueLOONG64_OpLOONG64MOVFstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVHUload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVHUload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHUload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2305,7 +2343,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHUload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHUload)
@@ -2315,7 +2353,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHUload(v *Value) bool {
 		return true
 	}
 	// match: (MOVHUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2327,7 +2365,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHUload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHUload)
@@ -2400,8 +2438,10 @@ func rewriteValueLOONG64_OpLOONG64MOVHUreg(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVHload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVHload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2412,7 +2452,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHload)
@@ -2422,7 +2462,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHload(v *Value) bool {
 		return true
 	}
 	// match: (MOVHload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2434,7 +2474,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHload)
@@ -2530,8 +2570,10 @@ func rewriteValueLOONG64_OpLOONG64MOVHstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVHstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2543,7 +2585,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHstore)
@@ -2553,7 +2595,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVHstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2566,7 +2608,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHstore)
@@ -2648,8 +2690,10 @@ func rewriteValueLOONG64_OpLOONG64MOVHstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVHstorezero(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVHstorezero [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHstorezero [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2660,7 +2704,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstorezero(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHstorezero)
@@ -2670,7 +2714,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstorezero(v *Value) bool {
 		return true
 	}
 	// match: (MOVHstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVHstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2682,7 +2726,7 @@ func rewriteValueLOONG64_OpLOONG64MOVHstorezero(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVHstorezero)
@@ -2696,8 +2740,10 @@ func rewriteValueLOONG64_OpLOONG64MOVHstorezero(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVVload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVVload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2708,7 +2754,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVload)
@@ -2718,7 +2764,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVload(v *Value) bool {
 		return true
 	}
 	// match: (MOVVload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2730,7 +2776,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVload)
@@ -2772,8 +2818,10 @@ func rewriteValueLOONG64_OpLOONG64MOVVstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVVstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2785,7 +2833,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVstore)
@@ -2795,7 +2843,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVVstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2808,7 +2856,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVstore)
@@ -2822,8 +2870,10 @@ func rewriteValueLOONG64_OpLOONG64MOVVstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVVstorezero(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVVstorezero [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVstorezero [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2834,7 +2884,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstorezero(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVstorezero)
@@ -2844,7 +2894,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstorezero(v *Value) bool {
 		return true
 	}
 	// match: (MOVVstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVVstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2856,7 +2906,7 @@ func rewriteValueLOONG64_OpLOONG64MOVVstorezero(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVVstorezero)
@@ -2870,8 +2920,10 @@ func rewriteValueLOONG64_OpLOONG64MOVVstorezero(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVWUload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVWUload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWUload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2882,7 +2934,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWUload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWUload)
@@ -2892,7 +2944,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWUload(v *Value) bool {
 		return true
 	}
 	// match: (MOVWUload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWUload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -2904,7 +2956,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWUload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWUload)
@@ -2999,8 +3051,10 @@ func rewriteValueLOONG64_OpLOONG64MOVWUreg(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVWload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVWload [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWload [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3011,7 +3065,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWload(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWload)
@@ -3021,7 +3075,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWload(v *Value) bool {
 		return true
 	}
 	// match: (MOVWload [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWload [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3033,7 +3087,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWload(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWload)
@@ -3162,8 +3216,10 @@ func rewriteValueLOONG64_OpLOONG64MOVWstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVWstore [off1] {sym} (ADDVconst [off2] ptr) val mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWstore [off1+int32(off2)] {sym} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3175,7 +3231,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWstore)
@@ -3185,7 +3241,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstore(v *Value) bool {
 		return true
 	}
 	// match: (MOVWstore [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) val mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWstore [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr val mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3198,7 +3254,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstore(v *Value) bool {
 		ptr := v_0.Args[0]
 		val := v_1
 		mem := v_2
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWstore)
@@ -3246,8 +3302,10 @@ func rewriteValueLOONG64_OpLOONG64MOVWstore(v *Value) bool {
 func rewriteValueLOONG64_OpLOONG64MOVWstorezero(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
 	// match: (MOVWstorezero [off1] {sym} (ADDVconst [off2] ptr) mem)
-	// cond: is32Bit(int64(off1)+off2)
+	// cond: is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWstorezero [off1+int32(off2)] {sym} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3258,7 +3316,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstorezero(v *Value) bool {
 		off2 := auxIntToInt64(v_0.AuxInt)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(is32Bit(int64(off1) + off2)) {
+		if !(is32Bit(int64(off1)+off2) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWstorezero)
@@ -3268,7 +3326,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstorezero(v *Value) bool {
 		return true
 	}
 	// match: (MOVWstorezero [off1] {sym1} (MOVVaddr [off2] {sym2} ptr) mem)
-	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2))
+	// cond: canMergeSym(sym1,sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)
 	// result: (MOVWstorezero [off1+int32(off2)] {mergeSym(sym1,sym2)} ptr mem)
 	for {
 		off1 := auxIntToInt32(v.AuxInt)
@@ -3280,7 +3338,7 @@ func rewriteValueLOONG64_OpLOONG64MOVWstorezero(v *Value) bool {
 		sym2 := auxToSym(v_0.Aux)
 		ptr := v_0.Args[0]
 		mem := v_1
-		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2))) {
+		if !(canMergeSym(sym1, sym2) && is32Bit(int64(off1)+int64(off2)) && (ptr.Op != OpSB || !config.ctxt.Flag_dynlink)) {
 			break
 		}
 		v.reset(OpLOONG64MOVWstorezero)
@@ -7570,6 +7628,7 @@ func rewriteValueLOONG64_OpZero(v *Value) bool {
 	return false
 }
 func rewriteBlockLOONG64(b *Block) bool {
+	typ := &b.Func.Config.Types
 	switch b.Kind {
 	case BlockLOONG64EQ:
 		// match: (EQ (FPFlagTrue cmp) yes no)
@@ -7769,10 +7828,12 @@ func rewriteBlockLOONG64(b *Block) bool {
 		}
 	case BlockIf:
 		// match: (If cond yes no)
-		// result: (NE cond yes no)
+		// result: (NE (MOVBUreg <typ.UInt64> cond) yes no)
 		for {
 			cond := b.Controls[0]
-			b.resetWithControl(BlockLOONG64NE, cond)
+			v0 := b.NewValue0(cond.Pos, OpLOONG64MOVBUreg, typ.UInt64)
+			v0.AddArg(cond)
+			b.resetWithControl(BlockLOONG64NE, v0)
 			return true
 		}
 	case BlockLOONG64LEZ:

src/cmd/compile/internal/ssa/rewritePPC64latelower.go

@@ -11,6 +11,8 @@ func rewriteValuePPC64latelower(v *Value) bool {
 		return rewriteValuePPC64latelower_OpPPC64ADD(v)
 	case OpPPC64AND:
 		return rewriteValuePPC64latelower_OpPPC64AND(v)
+	case OpPPC64CMPconst:
+		return rewriteValuePPC64latelower_OpPPC64CMPconst(v)
 	case OpPPC64ISEL:
 		return rewriteValuePPC64latelower_OpPPC64ISEL(v)
 	case OpPPC64RLDICL:
@@ -144,6 +146,361 @@ func rewriteValuePPC64latelower_OpPPC64AND(v *Value) bool {
 	}
 	return false
 }
+func rewriteValuePPC64latelower_OpPPC64CMPconst(v *Value) bool {
+	v_0 := v.Args[0]
+	// match: (CMPconst [0] z:(ADD x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64ADD {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(AND x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64AND {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(ANDN x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64ANDN {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(OR x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64OR {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(SUB x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64SUB {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(NOR x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64NOR {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(XOR x y))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64XOR {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(NEG x))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64NEG {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(CNTLZD x))
+	// cond: v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64CNTLZD {
+			break
+		}
+		if !(v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst [0] z:(ADDconst [c] x))
+	// cond: int64(int16(c)) == c && v.Block == z.Block
+	// result: (CMPconst [0] convertPPC64OpToOpCC(z))
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		z := v_0
+		if z.Op != OpPPC64ADDconst {
+			break
+		}
+		c := auxIntToInt64(z.AuxInt)
+		if !(int64(int16(c)) == c && v.Block == z.Block) {
+			break
+		}
+		v.reset(OpPPC64CMPconst)
+		v.AuxInt = int64ToAuxInt(0)
+		v.AddArg(convertPPC64OpToOpCC(z))
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(ADDCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64ADDCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(ANDCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64ANDCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(ANDNCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64ANDNCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(ORCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64ORCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(SUBCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64SUBCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(NORCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64NORCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(XORCC x y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64XORCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(ADDCCconst y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64ADDCCconst {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(NEGCC y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64NEGCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	// match: (CMPconst <t> [0] (Select0 z:(CNTLZDCC y)))
+	// result: (Select1 <t> z)
+	for {
+		t := v.Type
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpSelect0 {
+			break
+		}
+		z := v_0.Args[0]
+		if z.Op != OpPPC64CNTLZDCC {
+			break
+		}
+		v.reset(OpSelect1)
+		v.Type = t
+		v.AddArg(z)
+		return true
+	}
+	return false
+}
 func rewriteValuePPC64latelower_OpPPC64ISEL(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]

src/cmd/compile/internal/ssagen/ssa.go

@@ -5298,11 +5298,11 @@ func (s *state) callAddr(n *ir.CallExpr, k callKind) *ssa.Value {
 // Returns the address of the return value (or nil if none).
 func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool, deferExtra ir.Expr) *ssa.Value {
 	s.prevCall = nil
-	var callee *ir.Name    // target function (if static)
-	var closure *ssa.Value // ptr to closure to run (if dynamic)
-	var codeptr *ssa.Value // ptr to target code (if dynamic)
-	var dextra *ssa.Value  // defer extra arg
-	var rcvr *ssa.Value    // receiver to set
+	var calleeLSym *obj.LSym // target function (if static)
+	var closure *ssa.Value   // ptr to closure to run (if dynamic)
+	var codeptr *ssa.Value   // ptr to target code (if dynamic)
+	var dextra *ssa.Value    // defer extra arg
+	var rcvr *ssa.Value      // receiver to set
 	fn := n.Fun
 	var ACArgs []*types.Type    // AuxCall args
 	var ACResults []*types.Type // AuxCall results
@@ -5318,7 +5318,7 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool, deferExt
 	case ir.OCALLFUNC:
 		if (k == callNormal || k == callTail) && fn.Op() == ir.ONAME && fn.(*ir.Name).Class == ir.PFUNC {
 			fn := fn.(*ir.Name)
-			callee = fn
+			calleeLSym = callTargetLSym(fn)
 			if buildcfg.Experiment.RegabiArgs {
 				// This is a static call, so it may be
 				// a direct call to a non-ABIInternal
@@ -5466,8 +5466,8 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool, deferExt
 			// Note that the "receiver" parameter is nil because the actual receiver is the first input parameter.
 			aux := ssa.InterfaceAuxCall(params)
 			call = s.newValue1A(ssa.OpInterLECall, aux.LateExpansionResultType(), aux, codeptr)
-		case callee != nil:
-			aux := ssa.StaticAuxCall(callTargetLSym(callee), params)
+		case calleeLSym != nil:
+			aux := ssa.StaticAuxCall(calleeLSym, params)
 			call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux)
 			if k == callTail {
 				call.Op = ssa.OpTailLECall
@@ -7578,9 +7578,9 @@ func genssa(f *ssa.Func, pp *objw.Progs) {
 		for i, b := range f.Blocks {
 			idToIdx[b.ID] = i
 		}
-		// Note that at this moment, Prog.Pc is a sequence number; it's
-		// not a real PC until after assembly, so this mapping has to
-		// be done later.
+		// Register a callback that will be used later to fill in PCs into location
+		// lists. At the moment, Prog.Pc is a sequence number; it's not a real PC
+		// until after assembly, so the translation needs to be deferred.
 		debugInfo.GetPC = func(b, v ssa.ID) int64 {
 			switch v {
 			case ssa.BlockStart.ID:

src/cmd/compile/internal/syntax/type.go

@@ -39,25 +39,27 @@ type TypeAndValue struct {
 	exprFlags
 }
 
-type exprFlags uint8
+type exprFlags uint16
 
-func (f exprFlags) IsVoid() bool      { return f&1 != 0 }
-func (f exprFlags) IsType() bool      { return f&2 != 0 }
-func (f exprFlags) IsBuiltin() bool   { return f&4 != 0 }
-func (f exprFlags) IsValue() bool     { return f&8 != 0 }
-func (f exprFlags) IsNil() bool       { return f&16 != 0 }
-func (f exprFlags) Addressable() bool { return f&32 != 0 }
-func (f exprFlags) Assignable() bool  { return f&64 != 0 }
-func (f exprFlags) HasOk() bool       { return f&128 != 0 }
+func (f exprFlags) IsVoid() bool          { return f&1 != 0 }
+func (f exprFlags) IsType() bool          { return f&2 != 0 }
+func (f exprFlags) IsBuiltin() bool       { return f&4 != 0 } // a language builtin that resembles a function call, e.g., "make, append, new"
+func (f exprFlags) IsValue() bool         { return f&8 != 0 }
+func (f exprFlags) IsNil() bool           { return f&16 != 0 }
+func (f exprFlags) Addressable() bool     { return f&32 != 0 }
+func (f exprFlags) Assignable() bool      { return f&64 != 0 }
+func (f exprFlags) HasOk() bool           { return f&128 != 0 }
+func (f exprFlags) IsRuntimeHelper() bool { return f&256 != 0 } // a runtime function called from transformed syntax
 
-func (f *exprFlags) SetIsVoid()      { *f |= 1 }
-func (f *exprFlags) SetIsType()      { *f |= 2 }
-func (f *exprFlags) SetIsBuiltin()   { *f |= 4 }
-func (f *exprFlags) SetIsValue()     { *f |= 8 }
-func (f *exprFlags) SetIsNil()       { *f |= 16 }
-func (f *exprFlags) SetAddressable() { *f |= 32 }
-func (f *exprFlags) SetAssignable()  { *f |= 64 }
-func (f *exprFlags) SetHasOk()       { *f |= 128 }
+func (f *exprFlags) SetIsVoid()          { *f |= 1 }
+func (f *exprFlags) SetIsType()          { *f |= 2 }
+func (f *exprFlags) SetIsBuiltin()       { *f |= 4 }
+func (f *exprFlags) SetIsValue()         { *f |= 8 }
+func (f *exprFlags) SetIsNil()           { *f |= 16 }
+func (f *exprFlags) SetAddressable()     { *f |= 32 }
+func (f *exprFlags) SetAssignable()      { *f |= 64 }
+func (f *exprFlags) SetHasOk()           { *f |= 128 }
+func (f *exprFlags) SetIsRuntimeHelper() { *f |= 256 }
 
 // a typeAndValue contains the results of typechecking an expression.
 // It is embedded in expression nodes.