From 4b8895e2ddb8b9aa324417a0d01e6c09c9822e75 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 2 Dec 2016 16:22:45 -0800 Subject: [PATCH] [dev.inline] cmd/compile/internal/syntax: remove gcCompat uses in scanner - make the scanner unconditionally gc compatible - consistently use "invalid" instead "illegal" in errors Reviewed in and cherry-picked from https://go-review.googlesource.com/#/c/33896/. Change-Id: I4c4253e7392f3311b0d838bbe503576c9469b203 Reviewed-on: https://go-review.googlesource.com/34237 Run-TryBot: Robert Griesemer TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/syntax/parser.go | 1 - src/cmd/compile/internal/syntax/scanner.go | 48 ++++++++----------- .../compile/internal/syntax/scanner_test.go | 8 ++-- test/fixedbugs/issue11610.go | 2 +- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/src/cmd/compile/internal/syntax/parser.go b/src/cmd/compile/internal/syntax/parser.go index 505031ac2b..a58513478b 100644 --- a/src/cmd/compile/internal/syntax/parser.go +++ b/src/cmd/compile/internal/syntax/parser.go @@ -53,7 +53,6 @@ func (p *parser) init(filename string, src io.Reader, errh ErrorHandler, pragh P p.pragma |= pragh(p.pos_at(line, col), text) } }, - gcCompat, ) p.first = nil diff --git a/src/cmd/compile/internal/syntax/scanner.go b/src/cmd/compile/internal/syntax/scanner.go index 8af2f1ce14..149827b21c 100644 --- a/src/cmd/compile/internal/syntax/scanner.go +++ b/src/cmd/compile/internal/syntax/scanner.go @@ -21,9 +21,8 @@ import ( type scanner struct { source - pragh func(line, col uint, msg string) - gcCompat bool // TODO(gri) remove this eventually (only here so we can build w/o parser) - nlsemi bool // if set '\n' and EOF translate to ';' + pragh func(line, col uint, msg string) + nlsemi bool // if set '\n' and EOF translate to ';' // current token, valid after calling next() line, col uint @@ -34,10 +33,9 @@ type scanner struct { prec int // valid if tok is _Operator, _AssignOp, or _IncOp } -func (s *scanner) init(src io.Reader, errh, pragh func(line, col uint, msg string), gcCompat bool) { +func (s *scanner) init(src io.Reader, errh, pragh func(line, col uint, msg string)) { s.source.init(src, errh) s.pragh = pragh - s.gcCompat = gcCompat s.nlsemi = false } @@ -67,7 +65,7 @@ redo: // token start s.line, s.col = s.source.line0, s.source.col0 - if isLetter(c) || c >= utf8.RuneSelf && (unicode.IsLetter(c) || s.isCompatRune(c, true)) { + if isLetter(c) || c >= utf8.RuneSelf && s.isIdentRune(c, true) { s.ident() return } @@ -290,7 +288,7 @@ redo: default: s.tok = 0 - s.error(fmt.Sprintf("illegal character %#U", c)) + s.error(fmt.Sprintf("invalid character %#U", c)) goto redo } @@ -324,7 +322,7 @@ func (s *scanner) ident() { // general case if c >= utf8.RuneSelf { - for unicode.IsLetter(c) || c == '_' || unicode.IsDigit(c) || s.isCompatRune(c, false) { + for s.isIdentRune(c, false) { c = s.getr() } } @@ -346,14 +344,18 @@ func (s *scanner) ident() { s.tok = _Name } -func (s *scanner) isCompatRune(c rune, start bool) bool { - if !s.gcCompat || c < utf8.RuneSelf { - return false - } - if start && unicode.IsNumber(c) { - s.error(fmt.Sprintf("identifier cannot begin with digit %#U", c)) - } else { +func (s *scanner) isIdentRune(c rune, first bool) bool { + switch { + case unicode.IsLetter(c) || c == '_': + // ok + case unicode.IsDigit(c): + if first { + s.error(fmt.Sprintf("identifier cannot begin with digit %#U", c)) + } + case c >= utf8.RuneSelf: s.error(fmt.Sprintf("invalid identifier character %#U", c)) + default: + return false } return true } @@ -643,19 +645,11 @@ func (s *scanner) escape(quote rune) bool { if c < 0 { return true // complain in caller about EOF } - if s.gcCompat { - name := "hex" - if base == 8 { - name = "octal" - } - s.error(fmt.Sprintf("non-%s character in escape sequence: %c", name, c)) - } else { - if c != quote { - s.error(fmt.Sprintf("illegal character %#U in escape sequence", c)) - } else { - s.error("escape sequence incomplete") - } + kind := "hex" + if base == 8 { + kind = "octal" } + s.error(fmt.Sprintf("non-%s character in escape sequence: %c", kind, c)) s.ungetr() return false } diff --git a/src/cmd/compile/internal/syntax/scanner_test.go b/src/cmd/compile/internal/syntax/scanner_test.go index 988a74c287..5532780399 100644 --- a/src/cmd/compile/internal/syntax/scanner_test.go +++ b/src/cmd/compile/internal/syntax/scanner_test.go @@ -22,7 +22,7 @@ func TestScanner(t *testing.T) { defer src.Close() var s scanner - s.init(src, nil, nil, false) + s.init(src, nil, nil) for { s.next() if s.tok == _EOF { @@ -51,7 +51,7 @@ func TestTokens(t *testing.T) { // scan source var got scanner - got.init(&bytesReader{buf}, nil, nil, false) + got.init(&bytesReader{buf}, nil, nil) got.next() for i, want := range sampleTokens { nlsemi := false @@ -269,7 +269,7 @@ func TestScanErrors(t *testing.T) { // token-level errors {"x + ~y", "bitwise complement operator is ^", 1, 5}, - {"foo$bar = 0", "illegal character U+0024 '$'", 1, 4}, + {"foo$bar = 0", "invalid character U+0024 '$'", 1, 4}, {"const x = 0xyz", "malformed hex constant", 1, 13}, {"0123456789", "malformed octal constant", 1, 11}, {"0123456789. /* foobar", "comment not terminated", 1, 13}, // valid float constant @@ -348,7 +348,7 @@ func TestScanErrors(t *testing.T) { // TODO(gri) make this use position info t.Errorf("%q: got unexpected %q at line = %d", test.src, msg, line) } - }, nil, true) + }, nil) for { s.next() diff --git a/test/fixedbugs/issue11610.go b/test/fixedbugs/issue11610.go index 5e77932362..8ca31bf394 100644 --- a/test/fixedbugs/issue11610.go +++ b/test/fixedbugs/issue11610.go @@ -9,7 +9,7 @@ package a import"" // ERROR "import path is empty" -var? // ERROR "illegal character U\+003F '\?'" +var? // ERROR "invalid character U\+003F '\?'" var x int // ERROR "unexpected var"