text/scanner: don't crash when calling TokenText in error handler

Make sure Scanner.tokEnd is set before we call Scanner.Error
and update documentation accordingly.
(Until now tokEnd was only set before returning from Scan,
so a call to TokenText during error handling may have crashed.)

While at it, tighten a check in Scanner.TokenText to ensure
Scanner.tokEnd >= Scanner.tokPos if we have a token.

Also, silence error messages to Stderr in unrelated TestIllegalExponent.

Fixes #29723.

Change-Id: Ia97beeae91eaf9e0ed3dada0a806f1f7122461cc
Reviewed-on: https://go-review.googlesource.com/c/157819
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

src/text/scanner/scanner.go

@@ -322,6 +322,7 @@ func (s *Scanner) Peek() rune {
 }
 
 func (s *Scanner) error(msg string) {
+	s.tokEnd = s.srcPos - s.lastCharLen // make sure token text is terminated
 	s.ErrorCount++
 	if s.Error != nil {
 		s.Error(s, msg)
@@ -664,17 +665,18 @@ func (s *Scanner) Pos() (pos Position) {
 }
 
 // TokenText returns the string corresponding to the most recently scanned token.
-// Valid after calling Scan().
+// Valid after calling Scan and in calls of Scanner.Error.
 func (s *Scanner) TokenText() string {
 	if s.tokPos < 0 {
 		// no token text
 		return ""
 	}
 
-	if s.tokEnd < 0 {
+	if s.tokEnd < s.tokPos {
 		// if EOF was reached, s.tokEnd is set to -1 (s.srcPos == 0)
 		s.tokEnd = s.tokPos
 	}
+	// s.tokEnd >= s.tokPos
 
 	if s.tokBuf.Len() == 0 {
 		// common case: the entire token text is still in srcBuf

src/text/scanner/scanner_test.go

@@ -293,6 +293,12 @@ func TestScan(t *testing.T) {
 func TestIllegalExponent(t *testing.T) {
 	const src = "1.5e 1.5E 1e+ 1e- 1.5z"
 	s := new(Scanner).Init(strings.NewReader(src))
+	s.Error = func(s *Scanner, msg string) {
+		const want = "illegal exponent"
+		if msg != want {
+			t.Errorf("%s: got error %q; want %q", s.TokenText(), msg, want)
+		}
+	}
 	checkTokErr(t, s, 1, Float, "1.5e")
 	checkTokErr(t, s, 1, Float, "1.5E")
 	checkTokErr(t, s, 1, Float, "1e+")
@@ -692,3 +698,16 @@ func TestScanEOFHandling(t *testing.T) {
 		t.Errorf("scanner called Read %d times, not once", r)
 	}
 }
+
+func TestIssue29723(t *testing.T) {
+	s := new(Scanner).Init(strings.NewReader(`x "`))
+	s.Error = func(s *Scanner, _ string) {
+		got := s.TokenText() // this call shouldn't panic
+		const want = `"`
+		if got != want {
+			t.Errorf("got %q; want %q", got, want)
+		}
+	}
+	for r := s.Scan(); r != EOF; r = s.Scan() {
+	}
+}