[dev.typeparams] go/token, go/scanner: add the "~" operator

This is an approximate port of CL 307370 to go/token and go/scanner.

Change-Id: I5b789408f825f7e39f569322cb67802117b9d734
Reviewed-on: https://go-review.googlesource.com/c/go/+/324992
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/go/scanner/scanner.go

@@ -969,6 +969,8 @@ scanAgain:
 			}
 		case '|':
 			tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+		case '~':
+			tok = token.TILDE
 		default:
 			// next reports unexpected BOMs - don't repeat
 			if ch != bom {

src/go/scanner/scanner_test.go

@@ -40,7 +40,7 @@ type elt struct {
 	class int
 }
 
-var tokens = [...]elt{
+var tokens = []elt{
 	// Special tokens
 	{token.COMMENT, "/* a comment */", special},
 	{token.COMMENT, "// a comment \n", special},
@@ -149,6 +149,7 @@ var tokens = [...]elt{
 	{token.RBRACE, "}", operator},
 	{token.SEMICOLON, ";", operator},
 	{token.COLON, ":", operator},
+	{token.TILDE, "~", operator},
 
 	// Keywords
 	{token.BREAK, "break", keyword},

src/go/token/token.go

@@ -125,6 +125,11 @@ const (
 	TYPE
 	VAR
 	keyword_end
+
+	additional_beg
+	// additional tokens, handled in an ad-hoc manner
+	TILDE
+	additional_end
 )
 
 var tokens = [...]string{
@@ -225,6 +230,8 @@ var tokens = [...]string{
 	SWITCH: "switch",
 	TYPE:   "type",
 	VAR:    "var",
+
+	TILDE: "~",
 }
 
 // String returns the string corresponding to the token tok.
@@ -304,7 +311,9 @@ func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_en
 // IsOperator returns true for tokens corresponding to operators and
 // delimiters; it returns false otherwise.
 //
-func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+func (tok Token) IsOperator() bool {
+	return (operator_beg < tok && tok < operator_end) || tok == TILDE
+}
 
 // IsKeyword returns true for tokens corresponding to keywords;
 // it returns false otherwise.