cmd/compile: fix ICE from invalid operations on float/complex constants

Typechecking treats all untyped numbers as integers for the purposes
of validating operators. However, when I refactoring constant
operation evalution in golang.org/cl/139901, I mistakenly interpreted
that the only invalid case that needed to be preserved was % (modulo)
on floating-point values.

This CL restores the other remaining cases that were dropped from that
CL. It also uses the phrasing "invalid operation" instead of "illegal
constant expression" for better consistency with the rest of
cmd/compile and with go/types.

Lastly, this CL extends setconst to recognize failed constant folding
(e.g., division by zero) so that we can properly mark those
expressions as broken rather than continuing forward with bogus values
that might lead to further spurious errors.

Fixes #31060.

Change-Id: I1ab6491371925e22bc8b95649f1a0eed010abca6
Reviewed-on: https://go-review.googlesource.com/c/go/+/169719
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/cmd/compile/internal/gc/const.go

@@ -838,15 +838,13 @@ Outer:
 		case ODIV:
 			if y.CmpInt64(0) == 0 {
 				yyerror("division by zero")
-				u.SetOverflow()
-				break
+				return Val{}
 			}
 			u.Quo(y)
 		case OMOD:
 			if y.CmpInt64(0) == 0 {
 				yyerror("division by zero")
-				u.SetOverflow()
-				break
+				return Val{}
 			}
 			u.Rem(y)
 		case OOR:
@@ -877,13 +875,13 @@ Outer:
 		case ODIV:
 			if y.CmpFloat64(0) == 0 {
 				yyerror("division by zero")
-				u.SetFloat64(1)
-				break
+				return Val{}
 			}
 			u.Quo(y)
-		case OMOD:
-			// TODO(mdempsky): Move to typecheck.
-			yyerror("illegal constant expression: floating-point %% operation")
+		case OMOD, OOR, OAND, OANDNOT, OXOR:
+			// TODO(mdempsky): Move to typecheck; see #31060.
+			yyerror("invalid operation: operator %v not defined on untyped float", op)
+			return Val{}
 		default:
 			break Outer
 		}
@@ -907,9 +905,12 @@ Outer:
 		case ODIV:
 			if !u.Div(y) {
 				yyerror("complex division by zero")
-				u.Real.SetFloat64(1)
-				u.Imag.SetFloat64(0)
+				return Val{}
 			}
+		case OMOD, OOR, OAND, OANDNOT, OXOR:
+			// TODO(mdempsky): Move to typecheck; see #31060.
+			yyerror("invalid operation: operator %v not defined on untyped complex", op)
+			return Val{}
 		default:
 			break Outer
 		}
@@ -956,19 +957,31 @@ func unaryOp(op Op, x Val, t *types.Type) Val {
 		}
 
 	case OBITNOT:
-		x := x.U.(*Mpint)
+		switch x.Ctype() {
+		case CTINT, CTRUNE:
+			x := x.U.(*Mpint)
 
-		u := new(Mpint)
-		u.Rune = x.Rune
-		if t.IsSigned() || t.IsUntyped() {
-			// Signed values change sign.
-			u.SetInt64(-1)
-		} else {
-			// Unsigned values invert their bits.
-			u.Set(maxintval[t.Etype])
+			u := new(Mpint)
+			u.Rune = x.Rune
+			if t.IsSigned() || t.IsUntyped() {
+				// Signed values change sign.
+				u.SetInt64(-1)
+			} else {
+				// Unsigned values invert their bits.
+				u.Set(maxintval[t.Etype])
+			}
+			u.Xor(x)
+			return Val{U: u}
+
+		case CTFLT:
+			// TODO(mdempsky): Move to typecheck; see #31060.
+			yyerror("invalid operation: operator %v not defined on untyped float", op)
+			return Val{}
+		case CTCPLX:
+			// TODO(mdempsky): Move to typecheck; see #31060.
+			yyerror("invalid operation: operator %v not defined on untyped complex", op)
+			return Val{}
 		}
-		u.Xor(x)
-		return Val{U: u}
 
 	case ONOT:
 		return Val{U: !x.U.(bool)}
@@ -1001,6 +1014,12 @@ func shiftOp(x Val, op Op, y Val) Val {
 
 // setconst rewrites n as an OLITERAL with value v.
 func setconst(n *Node, v Val) {
+	// If constant folding failed, mark n as broken and give up.
+	if v.U == nil {
+		n.Type = nil
+		return
+	}
+
 	// Ensure n.Orig still points to a semantically-equivalent
 	// expression after we rewrite n into a constant.
 	if n.Orig == n {

test/const1.go

@@ -68,7 +68,7 @@ var (
 	c3 float64 = float64(Big) * Big // ERROR "overflow"
 	c4         = Big * Big          // ERROR "overflow"
 	c5         = Big / 0            // ERROR "division by zero"
-	c6         = 1000 % 1e3         // ERROR "floating-point % operation|expected integer type"
+	c6         = 1000 % 1e3         // ERROR "invalid operation|expected integer type"
 )
 
 func f(int)

test/fixedbugs/issue31060.go

@@ -0,0 +1,30 @@
+// errorcheck
+
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+const (
+	f = 1.0
+	c = 1.0i
+
+	_ = ^f // ERROR "invalid operation|expected integer"
+	_ = ^c // ERROR "invalid operation|expected integer"
+
+	_ = f % f // ERROR "invalid operation|expected integer"
+	_ = c % c // ERROR "invalid operation|expected integer"
+
+	_ = f & f // ERROR "invalid operation|expected integer"
+	_ = c & c // ERROR "invalid operation|expected integer"
+
+	_ = f | f // ERROR "invalid operation|expected integer"
+	_ = c | c // ERROR "invalid operation|expected integer"
+
+	_ = f ^ f // ERROR "invalid operation|expected integer"
+	_ = c ^ c // ERROR "invalid operation|expected integer"
+
+	_ = f &^ f // ERROR "invalid operation|expected integer"
+	_ = c &^ c // ERROR "invalid operation|expected integer"
+)