[dev.typeparams] cmd/compile/internal/types2: ensure that Named.check is nilled out once it is expanded

This is a port of
- https://golang.org/cl/318849
- https://golang.org/cl/322974

For #45580.

Change-Id: Ie0700ed6c8d472305d5ba7ff97da1ae063152aa3
Reviewed-on: https://go-review.googlesource.com/c/go/+/323030
Trust: Robert Griesemer <gri@golang.org>
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

src/cmd/compile/internal/types2/decl.go

@@ -523,15 +523,37 @@ func (check *Checker) varDecl(obj *Var, lhs []*Var, typ, init syntax.Expr) {
 // n0.check != nil, the cycle is reported.
 func (n0 *Named) under() Type {
 	u := n0.underlying
-	if u == nil {
-		return Typ[Invalid]
+
+	if u == Typ[Invalid] {
+		return u
 	}
 
 	// If the underlying type of a defined type is not a defined
-	// type, then that is the desired underlying type.
+	// (incl. instance) type, then that is the desired underlying
+	// type.
+	switch u.(type) {
+	case nil:
+		return Typ[Invalid]
+	default:
+		// common case
+		return u
+	case *Named, *instance:
+		// handled below
+	}
+
+	if n0.check == nil {
+		panic("internal error: Named.check == nil but type is incomplete")
+	}
+
+	// Invariant: after this point n0 as well as any named types in its
+	// underlying chain should be set up when this function exits.
+	check := n0.check
+
+	// If we can't expand u at this point, it is invalid.
 	n := asNamed(u)
 	if n == nil {
-		return u // common case
+		n0.underlying = Typ[Invalid]
+		return n0.underlying
 	}
 
 	// Otherwise, follow the forward chain.
@@ -543,7 +565,16 @@ func (n0 *Named) under() Type {
 			u = Typ[Invalid]
 			break
 		}
-		n1 := asNamed(u)
+		var n1 *Named
+		switch u1 := u.(type) {
+		case *Named:
+			n1 = u1
+		case *instance:
+			n1, _ = u1.expand().(*Named)
+			if n1 == nil {
+				u = Typ[Invalid]
+			}
+		}
 		if n1 == nil {
 			break // end of chain
 		}
@@ -554,11 +585,7 @@ func (n0 *Named) under() Type {
 
 		if i, ok := seen[n]; ok {
 			// cycle
-			// TODO(gri) revert this to a method on Checker. Having a possibly
-			// nil Checker on Named and TypeParam is too subtle.
-			if n0.check != nil {
-				n0.check.cycleError(path[i:])
-			}
+			check.cycleError(path[i:])
 			u = Typ[Invalid]
 			break
 		}
@@ -568,8 +595,8 @@ func (n0 *Named) under() Type {
 		// We should never have to update the underlying type of an imported type;
 		// those underlying types should have been resolved during the import.
 		// Also, doing so would lead to a race condition (was issue #31749).
-		// Do this check always, not just in debug more (it's cheap).
-		if n0.check != nil && n.obj.pkg != n0.check.pkg {
+		// Do this check always, not just in debug mode (it's cheap).
+		if n.obj.pkg != check.pkg {
 			panic("internal error: imported type with unresolved underlying type")
 		}
 		n.underlying = u

src/cmd/compile/internal/types2/sanitize.go

@@ -134,6 +134,9 @@ func (s sanitizer) typ(typ Type) Type {
 		}
 
 	case *Named:
+		if debug && t.check != nil {
+			panic("internal error: Named.check != nil")
+		}
 		if orig := s.typ(t.fromRHS); orig != t.fromRHS {
 			t.fromRHS = orig
 		}

src/cmd/compile/internal/types2/type.go

@@ -544,7 +544,7 @@ func (c *Chan) Elem() Type { return c.elem }
 
 // A Named represents a named (defined) type.
 type Named struct {
-	check      *Checker    // for Named.under implementation
+	check      *Checker    // for Named.under implementation; nilled once under has been called
 	info       typeInfo    // for cycle detection
 	obj        *TypeName   // corresponding declared object
 	orig       *Named      // original, uninstantiated type
@@ -574,6 +574,23 @@ func (check *Checker) newNamed(obj *TypeName, orig *Named, underlying Type, tpar
 	if obj.typ == nil {
 		obj.typ = typ
 	}
+	// Ensure that typ is always expanded, at which point the check field can be
+	// nilled out.
+	//
+	// Note that currently we cannot nil out check inside typ.under(), because
+	// it's possible that typ is expanded multiple times.
+	//
+	// TODO(gri): clean this up so that under is the only function mutating
+	//            named types.
+	if check != nil {
+		check.later(func() {
+			switch typ.under().(type) {
+			case *Named, *instance:
+				panic("internal error: unexpanded underlying type")
+			}
+			typ.check = nil
+		})
+	}
 	return typ
 }