From 74242baa4136c7a9132a8ccd9881354442788c8c Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Tue, 11 May 2021 11:31:31 -0700 Subject: [PATCH 01/40] archive/zip: only preallocate File slice if reasonably sized Since the number of files in the EOCD record isn't validated, it isn't safe to preallocate Reader.Files using that field. A malformed archive can indicate it contains up to 1 << 128 - 1 files. We can still safely preallocate the slice by checking if the specified number of files in the archive is reasonable, given the size of the archive. Thanks to the OSS-Fuzz project for discovering this issue and to Emmanuel Odeke for reporting it. Fixes #46242 Fixes CVE-2021-33196 Change-Id: I3c76d8eec178468b380d87fdb4a3f2cb06f0ee76 Reviewed-on: https://go-review.googlesource.com/c/go/+/318909 Trust: Roland Shoemaker Trust: Katie Hockman Trust: Joe Tsai Run-TryBot: Roland Shoemaker TryBot-Result: Go Bot Reviewed-by: Katie Hockman Reviewed-by: Joe Tsai --- src/archive/zip/reader.go | 10 +++++- src/archive/zip/reader_test.go | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/archive/zip/reader.go b/src/archive/zip/reader.go index 808cf274ca..2d53f4c723 100644 --- a/src/archive/zip/reader.go +++ b/src/archive/zip/reader.go @@ -96,7 +96,15 @@ func (z *Reader) init(r io.ReaderAt, size int64) error { return err } z.r = r - z.File = make([]*File, 0, end.directoryRecords) + // Since the number of directory records is not validated, it is not + // safe to preallocate z.File without first checking that the specified + // number of files is reasonable, since a malformed archive may + // indicate it contains up to 1 << 128 - 1 files. Since each file has a + // header which will be _at least_ 30 bytes we can safely preallocate + // if (data size / 30) >= end.directoryRecords. + if (uint64(size)-end.directorySize)/30 >= end.directoryRecords { + z.File = make([]*File, 0, end.directoryRecords) + } z.Comment = end.comment rs := io.NewSectionReader(r, 0, size) if _, err = rs.Seek(int64(end.directoryOffset), io.SeekStart); err != nil { diff --git a/src/archive/zip/reader_test.go b/src/archive/zip/reader_test.go index 35e681ec69..37dafe6c8e 100644 --- a/src/archive/zip/reader_test.go +++ b/src/archive/zip/reader_test.go @@ -1325,3 +1325,62 @@ func TestReadDataDescriptor(t *testing.T) { }) } } + +func TestCVE202133196(t *testing.T) { + // Archive that indicates it has 1 << 128 -1 files, + // this would previously cause a panic due to attempting + // to allocate a slice with 1 << 128 -1 elements. + data := []byte{ + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x08, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, + 0x03, 0x62, 0x61, 0x65, 0x03, 0x04, 0x00, 0x00, + 0xff, 0xff, 0x50, 0x4b, 0x07, 0x08, 0xbe, 0x20, + 0x5c, 0x6c, 0x09, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x00, + 0x14, 0x00, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xbe, 0x20, 0x5c, 0x6c, 0x09, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x02, 0x03, 0x50, 0x4b, 0x06, 0x06, 0x2c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, + 0x00, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x31, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x4b, 0x06, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x50, + 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x00, + } + _, err := NewReader(bytes.NewReader(data), int64(len(data))) + if err != ErrFormat { + t.Fatalf("unexpected error, got: %v, want: %v", err, ErrFormat) + } + + // Also check that an archive containing a handful of empty + // files doesn't cause an issue + b := bytes.NewBuffer(nil) + w := NewWriter(b) + for i := 0; i < 5; i++ { + _, err := w.Create("") + if err != nil { + t.Fatalf("Writer.Create failed: %s", err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Writer.Close failed: %s", err) + } + r, err := NewReader(bytes.NewReader(b.Bytes()), int64(b.Len())) + if err != nil { + t.Fatalf("NewReader failed: %s", err) + } + if len(r.File) != 5 { + t.Errorf("Archive has unexpected number of files, got %d, want 5", len(r.File)) + } +} From d050238bb653711b47335583c5425c9efec30e4e Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Tue, 25 May 2021 10:33:02 +0200 Subject: [PATCH 02/40] doc/go1.17: fix formatting for time changes Also add a link to the time.Time type and adjust the wording a bit. Change-Id: I2f4210ada6d253eb5804e6327b2432487beb8a05 Reviewed-on: https://go-review.googlesource.com/c/go/+/321811 Trust: Tobias Klauser Reviewed-by: Ian Lance Taylor --- doc/go1.17.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index c2317a4035..bdde26bd10 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -558,9 +558,10 @@ Do not send CLs removing the interior tags from such phrases.
time

- time.Time now has a GoString - method that will return a more useful value for times when printed with - the "%#v" format specifier in the fmt package. + The Time type now has a + GoString method that + will return a more useful value for times when printed with the + %#v format specifier in the fmt package.

From e4615ad74d5becdd1fcee4879775a6d4118583c5 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 12 May 2021 23:04:25 -0400 Subject: [PATCH 03/40] math/big: move division into natdiv.go Code moved and functions reordered to be in a consistent top-down dependency order, but otherwise unchanged. First step toward commenting division algorithms. Change-Id: Ib5e604fb5b2867edff3a228ba4e57b5cb32c4137 Reviewed-on: https://go-review.googlesource.com/c/go/+/321077 Trust: Russ Cox Trust: Katie Hockman Trust: Robert Griesemer Run-TryBot: Russ Cox TryBot-Result: Go Bot Reviewed-by: Katie Hockman Reviewed-by: Robert Griesemer --- src/math/big/arith.go | 14 -- src/math/big/nat.go | 325 -------------------------------------- src/math/big/natdiv.go | 346 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+), 339 deletions(-) create mode 100644 src/math/big/natdiv.go diff --git a/src/math/big/arith.go b/src/math/big/arith.go index e1947936d4..8f55c195d4 100644 --- a/src/math/big/arith.go +++ b/src/math/big/arith.go @@ -267,20 +267,6 @@ func divWW(x1, x0, y, m Word) (q, r Word) { return Word(qq), Word(r0 >> s) } -func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) { - r = xn - if len(x) == 1 { - qq, rr := bits.Div(uint(r), uint(x[0]), uint(y)) - z[0] = Word(qq) - return Word(rr) - } - rec := reciprocalWord(y) - for i := len(z) - 1; i >= 0; i-- { - z[i], r = divWW(r, x[i], y, rec) - } - return r -} - // reciprocalWord return the reciprocal of the divisor. rec = floor(( _B^2 - 1 ) / u - _B). u = d1 << nlz(d1). func reciprocalWord(d1 Word) Word { u := uint(d1 << nlz(d1)) diff --git a/src/math/big/nat.go b/src/math/big/nat.go index bbd6c8850b..140c619c8c 100644 --- a/src/math/big/nat.go +++ b/src/math/big/nat.go @@ -631,48 +631,6 @@ func (z nat) mulRange(a, b uint64) nat { return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b)) } -// q = (x-r)/y, with 0 <= r < y -func (z nat) divW(x nat, y Word) (q nat, r Word) { - m := len(x) - switch { - case y == 0: - panic("division by zero") - case y == 1: - q = z.set(x) // result is x - return - case m == 0: - q = z[:0] // result is 0 - return - } - // m > 0 - z = z.make(m) - r = divWVW(z, 0, x, y) - q = z.norm() - return -} - -func (z nat) div(z2, u, v nat) (q, r nat) { - if len(v) == 0 { - panic("division by zero") - } - - if u.cmp(v) < 0 { - q = z[:0] - r = z2.set(u) - return - } - - if len(v) == 1 { - var r2 Word - q, r2 = z.divW(u, v[0]) - r = z2.setWord(r2) - return - } - - q, r = z.divLarge(z2, u, v) - return -} - // getNat returns a *nat of len n. The contents may not be zero. // The pool holds *nat to avoid allocation when converting to interface{}. func getNat(n int) *nat { @@ -693,276 +651,6 @@ func putNat(x *nat) { var natPool sync.Pool -// q = (uIn-r)/vIn, with 0 <= r < vIn -// Uses z as storage for q, and u as storage for r if possible. -// See Knuth, Volume 2, section 4.3.1, Algorithm D. -// Preconditions: -// len(vIn) >= 2 -// len(uIn) >= len(vIn) -// u must not alias z -func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { - n := len(vIn) - m := len(uIn) - n - - // D1. - shift := nlz(vIn[n-1]) - // do not modify vIn, it may be used by another goroutine simultaneously - vp := getNat(n) - v := *vp - shlVU(v, vIn, shift) - - // u may safely alias uIn or vIn, the value of uIn is used to set u and vIn was already used - u = u.make(len(uIn) + 1) - u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) - - // z may safely alias uIn or vIn, both values were used already - if alias(z, u) { - z = nil // z is an alias for u - cannot reuse - } - q = z.make(m + 1) - - if n < divRecursiveThreshold { - q.divBasic(u, v) - } else { - q.divRecursive(u, v) - } - putNat(vp) - - q = q.norm() - shrVU(u, u, shift) - r = u.norm() - - return q, r -} - -// divBasic performs word-by-word division of u by v. -// The quotient is written in pre-allocated q. -// The remainder overwrites input u. -// -// Precondition: -// - q is large enough to hold the quotient u / v -// which has a maximum length of len(u)-len(v)+1. -func (q nat) divBasic(u, v nat) { - n := len(v) - m := len(u) - n - - qhatvp := getNat(n + 1) - qhatv := *qhatvp - - // D2. - vn1 := v[n-1] - rec := reciprocalWord(vn1) - for j := m; j >= 0; j-- { - // D3. - qhat := Word(_M) - var ujn Word - if j+n < len(u) { - ujn = u[j+n] - } - if ujn != vn1 { - var rhat Word - qhat, rhat = divWW(ujn, u[j+n-1], vn1, rec) - - // x1 | x2 = q̂v_{n-2} - vn2 := v[n-2] - x1, x2 := mulWW(qhat, vn2) - // test if q̂v_{n-2} > br̂ + u_{j+n-2} - ujn2 := u[j+n-2] - for greaterThan(x1, x2, rhat, ujn2) { - qhat-- - prevRhat := rhat - rhat += vn1 - // v[n-1] >= 0, so this tests for overflow. - if rhat < prevRhat { - break - } - x1, x2 = mulWW(qhat, vn2) - } - } - - // D4. - // Compute the remainder u - (q̂*v) << (_W*j). - // The subtraction may overflow if q̂ estimate was off by one. - qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) - qhl := len(qhatv) - if j+qhl > len(u) && qhatv[n] == 0 { - qhl-- - } - c := subVV(u[j:j+qhl], u[j:], qhatv) - if c != 0 { - c := addVV(u[j:j+n], u[j:], v) - // If n == qhl, the carry from subVV and the carry from addVV - // cancel out and don't affect u[j+n]. - if n < qhl { - u[j+n] += c - } - qhat-- - } - - if j == m && m == len(q) && qhat == 0 { - continue - } - q[j] = qhat - } - - putNat(qhatvp) -} - -const divRecursiveThreshold = 100 - -// divRecursive performs word-by-word division of u by v. -// The quotient is written in pre-allocated z. -// The remainder overwrites input u. -// -// Precondition: -// - len(z) >= len(u)-len(v) -// -// See Burnikel, Ziegler, "Fast Recursive Division", Algorithm 1 and 2. -func (z nat) divRecursive(u, v nat) { - // Recursion depth is less than 2 log2(len(v)) - // Allocate a slice of temporaries to be reused across recursion. - recDepth := 2 * bits.Len(uint(len(v))) - // large enough to perform Karatsuba on operands as large as v - tmp := getNat(3 * len(v)) - temps := make([]*nat, recDepth) - z.clear() - z.divRecursiveStep(u, v, 0, tmp, temps) - for _, n := range temps { - if n != nil { - putNat(n) - } - } - putNat(tmp) -} - -// divRecursiveStep computes the division of u by v. -// - z must be large enough to hold the quotient -// - the quotient will overwrite z -// - the remainder will overwrite u -func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { - u = u.norm() - v = v.norm() - - if len(u) == 0 { - z.clear() - return - } - n := len(v) - if n < divRecursiveThreshold { - z.divBasic(u, v) - return - } - m := len(u) - n - if m < 0 { - return - } - - // Produce the quotient by blocks of B words. - // Division by v (length n) is done using a length n/2 division - // and a length n/2 multiplication for each block. The final - // complexity is driven by multiplication complexity. - B := n / 2 - - // Allocate a nat for qhat below. - if temps[depth] == nil { - temps[depth] = getNat(n) - } else { - *temps[depth] = temps[depth].make(B + 1) - } - - j := m - for j > B { - // Divide u[j-B:j+n] by vIn. Keep remainder in u - // for next block. - // - // The following property will be used (Lemma 2): - // if u = u1 << s + u0 - // v = v1 << s + v0 - // then floor(u1/v1) >= floor(u/v) - // - // Moreover, the difference is at most 2 if len(v1) >= len(u/v) - // We choose s = B-1 since len(v)-s >= B+1 >= len(u/v) - s := (B - 1) - // Except for the first step, the top bits are always - // a division remainder, so the quotient length is <= n. - uu := u[j-B:] - - qhat := *temps[depth] - qhat.clear() - qhat.divRecursiveStep(uu[s:B+n], v[s:], depth+1, tmp, temps) - qhat = qhat.norm() - // Adjust the quotient: - // u = u_h << s + u_l - // v = v_h << s + v_l - // u_h = q̂ v_h + rh - // u = q̂ (v - v_l) + rh << s + u_l - // After the above step, u contains a remainder: - // u = rh << s + u_l - // and we need to subtract q̂ v_l - // - // But it may be a bit too large, in which case q̂ needs to be smaller. - qhatv := tmp.make(3 * n) - qhatv.clear() - qhatv = qhatv.mul(qhat, v[:s]) - for i := 0; i < 2; i++ { - e := qhatv.cmp(uu.norm()) - if e <= 0 { - break - } - subVW(qhat, qhat, 1) - c := subVV(qhatv[:s], qhatv[:s], v[:s]) - if len(qhatv) > s { - subVW(qhatv[s:], qhatv[s:], c) - } - addAt(uu[s:], v[s:], 0) - } - if qhatv.cmp(uu.norm()) > 0 { - panic("impossible") - } - c := subVV(uu[:len(qhatv)], uu[:len(qhatv)], qhatv) - if c > 0 { - subVW(uu[len(qhatv):], uu[len(qhatv):], c) - } - addAt(z, qhat, j-B) - j -= B - } - - // Now u < (v< 0 { - subVW(qhat, qhat, 1) - c := subVV(qhatv[:s], qhatv[:s], v[:s]) - if len(qhatv) > s { - subVW(qhatv[s:], qhatv[s:], c) - } - addAt(u[s:], v[s:], 0) - } - } - if qhatv.cmp(u.norm()) > 0 { - panic("impossible") - } - c := subVV(u[0:len(qhatv)], u[0:len(qhatv)], qhatv) - if c > 0 { - c = subVW(u[len(qhatv):], u[len(qhatv):], c) - } - if c > 0 { - panic("impossible") - } - - // Done! - addAt(z, qhat.norm(), 0) -} - // Length of x in bits. x must be normalized. func (x nat) bitLen() int { if i := len(x) - 1; i >= 0 { @@ -1170,19 +858,6 @@ func (z nat) xor(x, y nat) nat { return z.norm() } -// greaterThan reports whether (x1<<_W + x2) > (y1<<_W + y2) -func greaterThan(x1, x2, y1, y2 Word) bool { - return x1 > y1 || x1 == y1 && x2 > y2 -} - -// modW returns x % d. -func (x nat) modW(d Word) (r Word) { - // TODO(agl): we don't actually need to store the q value. - var q nat - q = q.make(len(x)) - return divWVW(q, 0, x, d) -} - // random creates a random integer in [0..limit), using the space in z if // possible. n is the bit length of limit. func (z nat) random(rand *rand.Rand, limit nat, n int) nat { diff --git a/src/math/big/natdiv.go b/src/math/big/natdiv.go new file mode 100644 index 0000000000..1330990c2c --- /dev/null +++ b/src/math/big/natdiv.go @@ -0,0 +1,346 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package big + +import "math/bits" + +func (z nat) div(z2, u, v nat) (q, r nat) { + if len(v) == 0 { + panic("division by zero") + } + + if u.cmp(v) < 0 { + q = z[:0] + r = z2.set(u) + return + } + + if len(v) == 1 { + var r2 Word + q, r2 = z.divW(u, v[0]) + r = z2.setWord(r2) + return + } + + q, r = z.divLarge(z2, u, v) + return +} + +// q = (x-r)/y, with 0 <= r < y +func (z nat) divW(x nat, y Word) (q nat, r Word) { + m := len(x) + switch { + case y == 0: + panic("division by zero") + case y == 1: + q = z.set(x) // result is x + return + case m == 0: + q = z[:0] // result is 0 + return + } + // m > 0 + z = z.make(m) + r = divWVW(z, 0, x, y) + q = z.norm() + return +} + +// modW returns x % d. +func (x nat) modW(d Word) (r Word) { + // TODO(agl): we don't actually need to store the q value. + var q nat + q = q.make(len(x)) + return divWVW(q, 0, x, d) +} + +func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) { + r = xn + if len(x) == 1 { + qq, rr := bits.Div(uint(r), uint(x[0]), uint(y)) + z[0] = Word(qq) + return Word(rr) + } + rec := reciprocalWord(y) + for i := len(z) - 1; i >= 0; i-- { + z[i], r = divWW(r, x[i], y, rec) + } + return r +} + +// q = (uIn-r)/vIn, with 0 <= r < vIn +// Uses z as storage for q, and u as storage for r if possible. +// See Knuth, Volume 2, section 4.3.1, Algorithm D. +// Preconditions: +// len(vIn) >= 2 +// len(uIn) >= len(vIn) +// u must not alias z +func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { + n := len(vIn) + m := len(uIn) - n + + // D1. + shift := nlz(vIn[n-1]) + // do not modify vIn, it may be used by another goroutine simultaneously + vp := getNat(n) + v := *vp + shlVU(v, vIn, shift) + + // u may safely alias uIn or vIn, the value of uIn is used to set u and vIn was already used + u = u.make(len(uIn) + 1) + u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) + + // z may safely alias uIn or vIn, both values were used already + if alias(z, u) { + z = nil // z is an alias for u - cannot reuse + } + q = z.make(m + 1) + + if n < divRecursiveThreshold { + q.divBasic(u, v) + } else { + q.divRecursive(u, v) + } + putNat(vp) + + q = q.norm() + shrVU(u, u, shift) + r = u.norm() + + return q, r +} + +// divBasic performs word-by-word division of u by v. +// The quotient is written in pre-allocated q. +// The remainder overwrites input u. +// +// Precondition: +// - q is large enough to hold the quotient u / v +// which has a maximum length of len(u)-len(v)+1. +func (q nat) divBasic(u, v nat) { + n := len(v) + m := len(u) - n + + qhatvp := getNat(n + 1) + qhatv := *qhatvp + + // D2. + vn1 := v[n-1] + rec := reciprocalWord(vn1) + for j := m; j >= 0; j-- { + // D3. + qhat := Word(_M) + var ujn Word + if j+n < len(u) { + ujn = u[j+n] + } + if ujn != vn1 { + var rhat Word + qhat, rhat = divWW(ujn, u[j+n-1], vn1, rec) + + // x1 | x2 = q̂v_{n-2} + vn2 := v[n-2] + x1, x2 := mulWW(qhat, vn2) + // test if q̂v_{n-2} > br̂ + u_{j+n-2} + ujn2 := u[j+n-2] + for greaterThan(x1, x2, rhat, ujn2) { + qhat-- + prevRhat := rhat + rhat += vn1 + // v[n-1] >= 0, so this tests for overflow. + if rhat < prevRhat { + break + } + x1, x2 = mulWW(qhat, vn2) + } + } + + // D4. + // Compute the remainder u - (q̂*v) << (_W*j). + // The subtraction may overflow if q̂ estimate was off by one. + qhatv[n] = mulAddVWW(qhatv[0:n], v, qhat, 0) + qhl := len(qhatv) + if j+qhl > len(u) && qhatv[n] == 0 { + qhl-- + } + c := subVV(u[j:j+qhl], u[j:], qhatv) + if c != 0 { + c := addVV(u[j:j+n], u[j:], v) + // If n == qhl, the carry from subVV and the carry from addVV + // cancel out and don't affect u[j+n]. + if n < qhl { + u[j+n] += c + } + qhat-- + } + + if j == m && m == len(q) && qhat == 0 { + continue + } + q[j] = qhat + } + + putNat(qhatvp) +} + +// greaterThan reports whether (x1<<_W + x2) > (y1<<_W + y2) +func greaterThan(x1, x2, y1, y2 Word) bool { + return x1 > y1 || x1 == y1 && x2 > y2 +} + +const divRecursiveThreshold = 100 + +// divRecursive performs word-by-word division of u by v. +// The quotient is written in pre-allocated z. +// The remainder overwrites input u. +// +// Precondition: +// - len(z) >= len(u)-len(v) +// +// See Burnikel, Ziegler, "Fast Recursive Division", Algorithm 1 and 2. +func (z nat) divRecursive(u, v nat) { + // Recursion depth is less than 2 log2(len(v)) + // Allocate a slice of temporaries to be reused across recursion. + recDepth := 2 * bits.Len(uint(len(v))) + // large enough to perform Karatsuba on operands as large as v + tmp := getNat(3 * len(v)) + temps := make([]*nat, recDepth) + z.clear() + z.divRecursiveStep(u, v, 0, tmp, temps) + for _, n := range temps { + if n != nil { + putNat(n) + } + } + putNat(tmp) +} + +// divRecursiveStep computes the division of u by v. +// - z must be large enough to hold the quotient +// - the quotient will overwrite z +// - the remainder will overwrite u +func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { + u = u.norm() + v = v.norm() + + if len(u) == 0 { + z.clear() + return + } + n := len(v) + if n < divRecursiveThreshold { + z.divBasic(u, v) + return + } + m := len(u) - n + if m < 0 { + return + } + + // Produce the quotient by blocks of B words. + // Division by v (length n) is done using a length n/2 division + // and a length n/2 multiplication for each block. The final + // complexity is driven by multiplication complexity. + B := n / 2 + + // Allocate a nat for qhat below. + if temps[depth] == nil { + temps[depth] = getNat(n) + } else { + *temps[depth] = temps[depth].make(B + 1) + } + + j := m + for j > B { + // Divide u[j-B:j+n] by vIn. Keep remainder in u + // for next block. + // + // The following property will be used (Lemma 2): + // if u = u1 << s + u0 + // v = v1 << s + v0 + // then floor(u1/v1) >= floor(u/v) + // + // Moreover, the difference is at most 2 if len(v1) >= len(u/v) + // We choose s = B-1 since len(v)-s >= B+1 >= len(u/v) + s := (B - 1) + // Except for the first step, the top bits are always + // a division remainder, so the quotient length is <= n. + uu := u[j-B:] + + qhat := *temps[depth] + qhat.clear() + qhat.divRecursiveStep(uu[s:B+n], v[s:], depth+1, tmp, temps) + qhat = qhat.norm() + // Adjust the quotient: + // u = u_h << s + u_l + // v = v_h << s + v_l + // u_h = q̂ v_h + rh + // u = q̂ (v - v_l) + rh << s + u_l + // After the above step, u contains a remainder: + // u = rh << s + u_l + // and we need to subtract q̂ v_l + // + // But it may be a bit too large, in which case q̂ needs to be smaller. + qhatv := tmp.make(3 * n) + qhatv.clear() + qhatv = qhatv.mul(qhat, v[:s]) + for i := 0; i < 2; i++ { + e := qhatv.cmp(uu.norm()) + if e <= 0 { + break + } + subVW(qhat, qhat, 1) + c := subVV(qhatv[:s], qhatv[:s], v[:s]) + if len(qhatv) > s { + subVW(qhatv[s:], qhatv[s:], c) + } + addAt(uu[s:], v[s:], 0) + } + if qhatv.cmp(uu.norm()) > 0 { + panic("impossible") + } + c := subVV(uu[:len(qhatv)], uu[:len(qhatv)], qhatv) + if c > 0 { + subVW(uu[len(qhatv):], uu[len(qhatv):], c) + } + addAt(z, qhat, j-B) + j -= B + } + + // Now u < (v< 0 { + subVW(qhat, qhat, 1) + c := subVV(qhatv[:s], qhatv[:s], v[:s]) + if len(qhatv) > s { + subVW(qhatv[s:], qhatv[s:], c) + } + addAt(u[s:], v[s:], 0) + } + } + if qhatv.cmp(u.norm()) > 0 { + panic("impossible") + } + c := subVV(u[0:len(qhatv)], u[0:len(qhatv)], qhatv) + if c > 0 { + c = subVW(u[len(qhatv):], u[len(qhatv):], c) + } + if c > 0 { + panic("impossible") + } + + // Done! + addAt(z, qhat.norm(), 0) +} From bfd7798a6c756b22d7376db527339b41bf7f7327 Mon Sep 17 00:00:00 2001 From: tyltr Date: Wed, 26 May 2021 15:41:27 +0000 Subject: [PATCH 04/40] runtime,cmd/link/internal/ld: fix typos Change-Id: I558590cef7e2311aadbdcb4088033e350d3aae32 GitHub-Last-Rev: 513944a6238e0e32e2a2c266b70f7d50c9db508d GitHub-Pull-Request: golang/go#46389 Reviewed-on: https://go-review.googlesource.com/c/go/+/322809 Reviewed-by: Cherry Mui Reviewed-by: Ian Lance Taylor --- src/cmd/link/internal/ld/data.go | 2 +- src/runtime/malloc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 223df63d9d..70fbb9dc4e 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1550,7 +1550,7 @@ func (ctxt *Link) dodata(symGroupType []sym.SymKind) { if ctxt.HeadType == objabi.Haix && ctxt.LinkMode == LinkExternal { // These symbols must have the same alignment as their section. - // Otherwize, ld might change the layout of Go sections. + // Otherwise, ld might change the layout of Go sections. ldr.SetSymAlign(ldr.Lookup("runtime.data", 0), state.dataMaxAlign[sym.SDATA]) ldr.SetSymAlign(ldr.Lookup("runtime.bss", 0), state.dataMaxAlign[sym.SBSS]) } diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 81e5225883..2759bbdaf9 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -296,7 +296,7 @@ const ( // high addresses if viewed as unsigned). // // On aix/ppc64, this offset allows to keep the heapAddrBits to - // 48. Otherwize, it would be 60 in order to handle mmap addresses + // 48. Otherwise, it would be 60 in order to handle mmap addresses // (in range 0x0a00000000000000 - 0x0afffffffffffff). But in this // case, the memory reserved in (s *pageAlloc).init for chunks // is causing important slowdowns. From 39da9ae5130afa58f8b9e4ea609a57d516bd78db Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 6 May 2021 22:28:37 -0400 Subject: [PATCH 05/40] go/types: ensure that Named.check is nilled out once it is expanded To support lazy expansion of defined types, *Named holds on to a *Checker field, which can pin the *Checker in memory. This can have meaningful memory implications for applications that keep type information around. Ensure that the Checker field is nilled out for any Named types that are instantiated during the type checking pass, by deferring a clean up to 'later' boundaries. In testing this almost exactly offset the ~6% memory footprint increase I observed with 1.17. Fixes #45580 Change-Id: I8aa5bb777573a924afe36e79fa65f8729336bceb Reviewed-on: https://go-review.googlesource.com/c/go/+/318849 Trust: Robert Findley Trust: Robert Griesemer Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/types/decl.go | 53 ++++++++++++++++++++++++++++++---------- src/go/types/sanitize.go | 3 +++ src/go/types/type.go | 17 ++++++++++++- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/go/types/decl.go b/src/go/types/decl.go index 5f38a346ce..9211febc6d 100644 --- a/src/go/types/decl.go +++ b/src/go/types/decl.go @@ -577,15 +577,37 @@ func (check *Checker) varDecl(obj *Var, lhs []*Var, typ, init ast.Expr) { // n0.check != nil, the cycle is reported. func (n0 *Named) under() Type { u := n0.underlying - if u == nil { - return Typ[Invalid] + + if u == Typ[Invalid] { + return u } // If the underlying type of a defined type is not a defined - // type, then that is the desired underlying type. + // (incl. instance) type, then that is the desired underlying + // type. + switch u.(type) { + case nil: + return Typ[Invalid] + default: + // common case + return u + case *Named, *instance: + // handled below + } + + if n0.check == nil { + panic("internal error: Named.check == nil but type is incomplete") + } + + // Invariant: after this point n0 as well as any named types in its + // underlying chain should be set up when this function exits. + check := n0.check + + // If we can't expand u at this point, it is invalid. n := asNamed(u) if n == nil { - return u // common case + n0.underlying = Typ[Invalid] + return n0.underlying } // Otherwise, follow the forward chain. @@ -597,7 +619,16 @@ func (n0 *Named) under() Type { u = Typ[Invalid] break } - n1 := asNamed(u) + var n1 *Named + switch u1 := u.(type) { + case *Named: + n1 = u1 + case *instance: + n1, _ = u1.expand().(*Named) + if n1 == nil { + u = Typ[Invalid] + } + } if n1 == nil { break // end of chain } @@ -608,11 +639,7 @@ func (n0 *Named) under() Type { if i, ok := seen[n]; ok { // cycle - // TODO(rFindley) revert this to a method on Checker. Having a possibly - // nil Checker on Named and TypeParam is too subtle. - if n0.check != nil { - n0.check.cycleError(path[i:]) - } + check.cycleError(path[i:]) u = Typ[Invalid] break } @@ -622,8 +649,8 @@ func (n0 *Named) under() Type { // We should never have to update the underlying type of an imported type; // those underlying types should have been resolved during the import. // Also, doing so would lead to a race condition (was issue #31749). - // Do this check always, not just in debug more (it's cheap). - if n0.check != nil && n.obj.pkg != n0.check.pkg { + // Do this check always, not just in debug mode (it's cheap). + if n.obj.pkg != check.pkg { panic("internal error: imported type with unresolved underlying type") } n.underlying = u @@ -665,7 +692,7 @@ func (check *Checker) typeDecl(obj *TypeName, tdecl *ast.TypeSpec, def *Named) { } else { // defined type declaration - named := &Named{check: check, obj: obj} + named := check.newNamed(obj, nil, nil) def.setUnderlying(named) obj.typ = named // make sure recursive type declarations terminate diff --git a/src/go/types/sanitize.go b/src/go/types/sanitize.go index 5970ab38c7..727ec173ea 100644 --- a/src/go/types/sanitize.go +++ b/src/go/types/sanitize.go @@ -135,6 +135,9 @@ func (s sanitizer) typ(typ Type) Type { } case *Named: + if debug && t.check != nil { + panic("internal error: Named.check != nil") + } if orig := s.typ(t.orig); orig != t.orig { t.orig = orig } diff --git a/src/go/types/type.go b/src/go/types/type.go index 3303cfc077..3fdb2365a0 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -644,7 +644,7 @@ func (c *Chan) Elem() Type { return c.elem } // A Named represents a named (defined) type. type Named struct { - check *Checker // for Named.under implementation + check *Checker // for Named.under implementation; nilled once under has been called info typeInfo // for cycle detection obj *TypeName // corresponding declared object orig Type // type (on RHS of declaration) this *Named type is derived of (for cycle reporting) @@ -673,6 +673,21 @@ func (check *Checker) newNamed(obj *TypeName, underlying Type, methods []*Func) if obj.typ == nil { obj.typ = typ } + // Ensure that typ is always expanded, at which point the check field can be + // nilled out. + // + // Note that currently we cannot nil out check inside typ.under(), because + // it's possible that typ is expanded multiple times. + // + // TODO(rFindley): clean this up so that under is the only function mutating + // named types. + check.later(func() { + switch typ.under().(type) { + case *Named, *instance: + panic("internal error: unexpanded underlying type") + } + typ.check = nil + }) return typ } From 55aefbb268d7d33ebf300ed5b1e38e55c10c8070 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Wed, 19 May 2021 16:34:58 -0400 Subject: [PATCH 06/40] doc/go1.17: mention enabling frame pointer on all ARM64 Updates #44513. Change-Id: I43e95de0423779b3311d96c56f7c8c1cc5be27b6 Reviewed-on: https://go-review.googlesource.com/c/go/+/321311 Trust: Cherry Mui Reviewed-by: Jeremy Faller --- doc/go1.17.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index bdde26bd10..5448b2af97 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -68,6 +68,14 @@ Do not send CLs removing the interior tags from such phrases. OpenBSD.

+

ARM64

+ +

+ Go programs now maintain stack frame pointers on the 64-bit ARM + architecture on all operating systems. Previously it maintained + stack frame pointers only on Linux, macOS, and iOS. +

+

TODO: complete the Ports section

From a92460fd2f5537bbd91a713ced00731d429563ac Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Wed, 26 May 2021 12:18:32 -0400 Subject: [PATCH 07/40] doc/go1.17: add release notes for runtime/metrics package Updates #44513. Change-Id: I571a791e9c76371be3b3f1a323f1ea8ff485cf0f Reviewed-on: https://go-review.googlesource.com/c/go/+/322857 Trust: Cherry Mui Reviewed-by: Jeremy Faller Reviewed-by: Michael Knyszek --- doc/go1.17.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 5448b2af97..da50935a61 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -496,6 +496,16 @@ Do not send CLs removing the interior tags from such phrases.
+
runtime/metrics
+
+

+ New metrics were added that track total bytes and objects allocated and freed. + A new metric tracking the distribution of goroutine scheduling latencies was + also added. +

+
+
+
strconv

From 02beecb3974e9010d2deefaf09266286b0c6f408 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 May 2021 13:43:15 -0700 Subject: [PATCH 08/40] mime: document use of the Shared MIME-Info Database For #44513. Fixes #46013. Change-Id: I382603208aa94b66d5220cf0f418b8528a4e4148 Reviewed-on: https://go-review.googlesource.com/c/go/+/322892 Trust: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- doc/go1.17.html | 4 +++- src/mime/type.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index da50935a61..771e2a6c8d 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -408,7 +408,9 @@ Do not send CLs removing the interior tags from such phrases.

mime

- TODO: https://golang.org/cl/305230: support reading shared mime-info database on unix systems + On Unix systems, the table of MIME types is now read from the local system's + Shared MIME-info Database + when available.

diff --git a/src/mime/type.go b/src/mime/type.go index 9bbbf216a1..26424339af 100644 --- a/src/mime/type.go +++ b/src/mime/type.go @@ -96,9 +96,11 @@ func initMime() { // Extensions are looked up first case-sensitively, then case-insensitively. // // The built-in table is small but on unix it is augmented by the local -// system's mime.types file(s) if available under one or more of these -// names: +// system's MIME-info database or mime.types file(s) if available under one or +// more of these names: // +// /usr/local/share/mime/globs2 +// /usr/share/mime/globs2 // /etc/mime.types // /etc/apache2/mime.types // /etc/apache/mime.types From 0fbecece98977f6d0578cef1e8f3ae00a54c8ac4 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 May 2021 11:12:43 -0700 Subject: [PATCH 09/40] doc/go1.17: document syscall changes Fixes #46023 Change-Id: Ia63829d03afb0936862b16f0971175cbaa1095bc Reviewed-on: https://go-review.googlesource.com/c/go/+/322890 Trust: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- doc/go1.17.html | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index 771e2a6c8d..194d3a0cd8 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -550,11 +550,16 @@ Do not send CLs removing the interior tags from such phrases.
syscall

- TODO: https://golang.org/cl/295371: do not overflow key memory in GetQueuedCompletionStatus +

+ The GetQueuedCompletionStatus and + PostQueuedCompletionStatus + functions are now deprecated. These functions have incorrect signatures and are superseded by + equivalents in the golang.org/x/sys/windows package.

- TODO: https://golang.org/cl/313653: restore signal mask after setting foreground process group + On Unix-like systems, the process group of a child process is now set with signals blocked. + This avoids sending a SIGTTOU to the child when the parent is in a background process group.

From 1d5298d46a695219ab4622ae5aa59898459fd0f5 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 May 2021 11:28:01 -0700 Subject: [PATCH 10/40] doc/go1.17: document net/... changes For #44513. Fixes #46014. Fixes #46015. Fixes #46016. Fixes #46017. Change-Id: I356483d68d07159281dfe2ea1e49430ddf200973 Reviewed-on: https://go-review.googlesource.com/c/go/+/322891 Trust: Damien Neil Run-TryBot: Damien Neil Reviewed-by: Ian Lance Taylor --- doc/go1.17.html | 23 +++++++++++++++++------ src/os/pipe_test.go | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index 194d3a0cd8..8313c2bc57 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -418,15 +418,20 @@ Do not send CLs removing the interior tags from such phrases.
net

- TODO: https://golang.org/cl/272668: add IP.IsPrivate + The new method IP.IsPrivate reports whether an address is + a private IPv4 address according to RFC 1918 + or a local IPv6 address according RFC 4193.

- TODO: https://golang.org/cl/301709: make go resolver aware of network parameter + The Go DNS resolver now only sends one DNS query when resolving an address for an IPv4-only or IPv6-only network, + rather than querying for both address families.

- TODO: https://golang.org/cl/307030: make ErrClosed and ParseError implement net.Error + The ErrClosed sentinel error and + ParseError error type now implement + the net.Error interface.

@@ -441,7 +446,9 @@ Do not send CLs removing the interior tags from such phrases.

- TODO: https://golang.org/cl/235437: add to deadlines only when positive + Setting the Server + ReadTimeout or WriteTimeout fields to a negative value now indicates no timeout + rather than an immediate timeout.

@@ -454,7 +461,10 @@ Do not send CLs removing the interior tags from such phrases.

net/http/httptest

- TODO: https://golang.org/cl/308950: panic on non-3 digit (XXX) status code in Recorder.WriteHeader + ResponseRecorder.WriteHeader> + now panics when the provided code is not a valid three-digit HTTP status code. + This matches the behavior of ResponseWriter> + implementations in the net/http package.

@@ -462,7 +472,8 @@ Do not send CLs removing the interior tags from such phrases.
net/url

- TODO: https://golang.org/cl/314850: add Values.Has + The new method Values.Has + reports whether a query parameter is set.

diff --git a/src/os/pipe_test.go b/src/os/pipe_test.go index b663618502..b3d5380b8d 100644 --- a/src/os/pipe_test.go +++ b/src/os/pipe_test.go @@ -462,7 +462,7 @@ func TestFdReadRace(t *testing.T) { // Give the other goroutine a chance to enter the Read. // It doesn't matter if this occasionally fails, the test // will still pass, it just won't test anything. - time.Sleep(10 * time.Millisecond) + //time.Sleep(10 * time.Millisecond) r.Fd() // The bug was that Fd would hang until Read timed out. From a62c08734f8e2fc5333036e3a069a55288d1f674 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 May 2021 15:44:46 -0700 Subject: [PATCH 11/40] src/os: revert accidentally submitted change Change-Id: Ib34984a6bd0abc76266e8aac96f9f8ad8ae21d17 Reviewed-on: https://go-review.googlesource.com/c/go/+/322894 Trust: Damien Neil Run-TryBot: Damien Neil Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- src/os/pipe_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/pipe_test.go b/src/os/pipe_test.go index b3d5380b8d..b663618502 100644 --- a/src/os/pipe_test.go +++ b/src/os/pipe_test.go @@ -462,7 +462,7 @@ func TestFdReadRace(t *testing.T) { // Give the other goroutine a chance to enter the Read. // It doesn't matter if this occasionally fails, the test // will still pass, it just won't test anything. - //time.Sleep(10 * time.Millisecond) + time.Sleep(10 * time.Millisecond) r.Fd() // The bug was that Fd would hang until Read timed out. From 3075ffc93e962792ddf43b2a528ef19b1577ffb7 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 26 May 2021 15:17:27 -0700 Subject: [PATCH 12/40] os: deflake TestFdReadRace The test would hang if the call to Fd set the pipe to be non-blocking before the Read entered the first read system call. Avoid that problem by writing data to the pipe to wake up the read. For #24481 Fixes #44818 Change-Id: I0b798874c7b81e7308a38ebbf657efc4392ffacd Reviewed-on: https://go-review.googlesource.com/c/go/+/322893 Trust: Ian Lance Taylor Run-TryBot: Ian Lance Taylor TryBot-Result: Go Bot Reviewed-by: Damien Neil --- src/os/pipe_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/os/pipe_test.go b/src/os/pipe_test.go index b663618502..41a1e9c78a 100644 --- a/src/os/pipe_test.go +++ b/src/os/pipe_test.go @@ -442,12 +442,14 @@ func TestFdReadRace(t *testing.T) { defer r.Close() defer w.Close() - c := make(chan bool) + const count = 10 + + c := make(chan bool, 1) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() - var buf [10]byte + var buf [count]byte r.SetReadDeadline(time.Now().Add(time.Minute)) c <- true if _, err := r.Read(buf[:]); os.IsTimeout(err) { @@ -466,8 +468,9 @@ func TestFdReadRace(t *testing.T) { r.Fd() // The bug was that Fd would hang until Read timed out. - // If the bug is fixed, then closing r here will cause - // the Read to exit before the timeout expires. + // If the bug is fixed, then writing to w and closing r here + // will cause the Read to exit before the timeout expires. + w.Write(make([]byte, count)) r.Close() }() From 6ff0ae2aa4fdb9c6c267efb30927e87563387c49 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Thu, 13 May 2021 13:45:55 -0400 Subject: [PATCH 13/40] crypto/elliptic: fix typo in p521Point type name Change-Id: I6cab3624c875d9a70441a560e84f91c9b2df17b9 Reviewed-on: https://go-review.googlesource.com/c/go/+/320070 Trust: Filippo Valsorda Trust: Katie Hockman Run-TryBot: Filippo Valsorda Run-TryBot: Katie Hockman TryBot-Result: Go Bot Reviewed-by: Katie Hockman --- src/crypto/elliptic/p521.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/crypto/elliptic/p521.go b/src/crypto/elliptic/p521.go index ce74e0539c..3d355943ec 100644 --- a/src/crypto/elliptic/p521.go +++ b/src/crypto/elliptic/p521.go @@ -52,7 +52,7 @@ func (curve p521Curve) IsOnCurve(x, y *big.Int) bool { return x3.Equal(y2) == 1 } -type p512Point struct { +type p521Point struct { x, y, z *fiat.P521Element } @@ -67,7 +67,7 @@ func fiatP521ToBigInt(x *fiat.P521Element) *big.Int { // affineFromJacobian brings a point in Jacobian coordinates back to affine // coordinates, with (0, 0) representing infinity by convention. It also goes // back to big.Int values to match the exposed API. -func (curve p521Curve) affineFromJacobian(p *p512Point) (x, y *big.Int) { +func (curve p521Curve) affineFromJacobian(p *p521Point) (x, y *big.Int) { if p.z.IsZero() == 1 { return new(big.Int), new(big.Int) } @@ -99,17 +99,17 @@ func bigIntToFiatP521(x *big.Int) *fiat.P521Element { // jacobianFromAffine converts (x, y) affine coordinates into (x, y, z) Jacobian // coordinates. It also converts from big.Int to fiat, which is necessarily a // messy and variable-time operation, which we can't avoid due to the exposed API. -func (curve p521Curve) jacobianFromAffine(x, y *big.Int) *p512Point { +func (curve p521Curve) jacobianFromAffine(x, y *big.Int) *p521Point { // (0, 0) is by convention the point at infinity, which can't be represented // in affine coordinates, but is (0, 0, 0) in Jacobian. if x.Sign() == 0 && y.Sign() == 0 { - return &p512Point{ + return &p521Point{ x: new(fiat.P521Element), y: new(fiat.P521Element), z: new(fiat.P521Element), } } - return &p512Point{ + return &p521Point{ x: bigIntToFiatP521(x), y: bigIntToFiatP521(y), z: new(fiat.P521Element).One(), @@ -123,7 +123,7 @@ func (curve p521Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { } // addJacobian sets q = p1 + p2, and returns q. The points may overlap. -func (q *p512Point) addJacobian(p1, p2 *p512Point) *p512Point { +func (q *p521Point) addJacobian(p1, p2 *p521Point) *p521Point { // https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl z1IsZero := p1.z.IsZero() z2IsZero := p2.z.IsZero() @@ -189,7 +189,7 @@ func (curve p521Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) { } // doubleJacobian sets q = p + p, and returns q. The points may overlap. -func (q *p512Point) doubleJacobian(p *p512Point) *p512Point { +func (q *p521Point) doubleJacobian(p *p521Point) *p521Point { // https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b delta := new(fiat.P521Element).Square(p.z) gamma := new(fiat.P521Element).Square(p.y) @@ -230,11 +230,11 @@ func (q *p512Point) doubleJacobian(p *p512Point) *p512Point { func (curve p521Curve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) { B := curve.jacobianFromAffine(Bx, By) - p, t := &p512Point{ + p, t := &p521Point{ x: new(fiat.P521Element), y: new(fiat.P521Element), z: new(fiat.P521Element), - }, &p512Point{ + }, &p521Point{ x: new(fiat.P521Element), y: new(fiat.P521Element), z: new(fiat.P521Element), From 9bc52686da81b515cf3ad654dfb1a536fabceafa Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 25 May 2021 12:21:11 -0400 Subject: [PATCH 14/40] cmd/go,cmd/link: do not check for staleness in most tests Instead, check that stale packages in the standard library are not rebuilt when already present in the build cache, and are not installed implicitly when rebuilt. We retain the staleness checks for the runtime package in tests involving '-i', because those are guaranteed to fail anyway if the package is stale and the "stale" failure message is arguably clearer. They can be removed if/when we remove the '-i' flag, but the runtime package is less likely to become stale because it does not have cgo dependencies. Fixes #46347 Updates #33598 Updates #35459 Updates #41696 Change-Id: I7b0a808addd930f9f4911ff53ded62272af75a40 Reviewed-on: https://go-review.googlesource.com/c/go/+/322629 Trust: Bryan C. Mills Run-TryBot: Bryan C. Mills TryBot-Result: Go Bot Reviewed-by: Cherry Mui Reviewed-by: Jay Conrod --- ...build_package_not_stale_trailing_slash.txt | 13 ----- src/cmd/go/testdata/script/cgo_stale.txt | 39 ++++++++++++++ src/cmd/go/testdata/script/list_std_stale.txt | 31 ----------- .../go/testdata/script/list_std_vendor.txt | 32 ++++++++++++ .../testdata/script/test_race_install_cgo.txt | 2 - src/cmd/go/testdata/script/toolexec.txt | 6 +++ src/cmd/link/dwarf_test.go | 52 +++++++++++++------ 7 files changed, 112 insertions(+), 63 deletions(-) delete mode 100644 src/cmd/go/testdata/script/build_package_not_stale_trailing_slash.txt create mode 100644 src/cmd/go/testdata/script/cgo_stale.txt delete mode 100644 src/cmd/go/testdata/script/list_std_stale.txt create mode 100644 src/cmd/go/testdata/script/list_std_vendor.txt diff --git a/src/cmd/go/testdata/script/build_package_not_stale_trailing_slash.txt b/src/cmd/go/testdata/script/build_package_not_stale_trailing_slash.txt deleted file mode 100644 index 38a151ef1f..0000000000 --- a/src/cmd/go/testdata/script/build_package_not_stale_trailing_slash.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Tests Issue #12690 - -[gccgo] skip 'gccgo does not have GOROOT' - -! stale runtime -! stale os -! stale io - -env GOROOT=$GOROOT'/' - -! stale runtime -! stale os -! stale io \ No newline at end of file diff --git a/src/cmd/go/testdata/script/cgo_stale.txt b/src/cmd/go/testdata/script/cgo_stale.txt new file mode 100644 index 0000000000..9e46855ead --- /dev/null +++ b/src/cmd/go/testdata/script/cgo_stale.txt @@ -0,0 +1,39 @@ +# golang.org/issue/46347: a stale runtime/cgo should only force a single rebuild + +[!cgo] skip +[short] skip + + +# If we set a unique CGO_CFLAGS, the installed copy of runtime/cgo +# should be reported as stale. + +env CGO_CFLAGS=-DTestScript_cgo_stale=true +stale runtime/cgo + + +# If we then build a package that uses cgo, runtime/cgo should be rebuilt and +# cached with the new flag, but not installed to GOROOT (and thus still stale). + +env GOCACHE=$WORK/cache # Use a fresh cache to avoid interference between runs. + +go build -x . +stderr '[/\\]cgo'$GOEXE'["]? .* -importpath runtime/cgo' +stale runtime/cgo + + +# After runtime/cgo has been rebuilt and cached, it should not be rebuilt again +# even though it is still reported as stale. + +go build -x . +! stderr '[/\\]cgo'$GOEXE'["]? .* -importpath runtime/cgo' +stale runtime/cgo + + +-- go.mod -- +module example.com/m + +go 1.17 +-- m.go -- +package m + +import "C" diff --git a/src/cmd/go/testdata/script/list_std_stale.txt b/src/cmd/go/testdata/script/list_std_stale.txt deleted file mode 100644 index e5c1f334fd..0000000000 --- a/src/cmd/go/testdata/script/list_std_stale.txt +++ /dev/null @@ -1,31 +0,0 @@ -# https://golang.org/issue/44725: packages in std should not be reported as stale, -# regardless of whether they are listed from within or outside GOROOT/src. - -# Control case: net should not be stale at the start of the test, -# and should depend on vendor/golang.org/… instead of golang.org/…. - -! stale net - -go list -deps net -stdout '^vendor/golang.org/x/net' -! stdout '^golang.org/x/net' - -# Net should also not be stale when viewed from within GOROOT/src, -# and should still report the same package dependencies. - -cd $GOROOT/src -! stale net - -go list -deps net -stdout '^vendor/golang.org/x/net' -! stdout '^golang.org/x/net' - - -# However, 'go mod' and 'go get' subcommands should report the original module -# dependencies, not the vendored packages. - -[!net] stop - -env GOPROXY= -go mod why -m golang.org/x/net -stdout '^# golang.org/x/net\nnet\ngolang.org/x/net' diff --git a/src/cmd/go/testdata/script/list_std_vendor.txt b/src/cmd/go/testdata/script/list_std_vendor.txt new file mode 100644 index 0000000000..8f27cc1e8d --- /dev/null +++ b/src/cmd/go/testdata/script/list_std_vendor.txt @@ -0,0 +1,32 @@ +# https://golang.org/issue/44725: packages in std should have the same +# dependencies regardless of whether they are listed from within or outside +# GOROOT/src. + +# Control case: net, viewed from outside the 'std' module, +# should depend on vendor/golang.org/… instead of golang.org/…. + +go list -deps net +stdout '^vendor/golang.org/x/net' +! stdout '^golang.org/x/net' +cp stdout $WORK/net-deps.txt + + +# It should still report the same package dependencies when viewed from +# within GOROOT/src. + +cd $GOROOT/src + +go list -deps net +stdout '^vendor/golang.org/x/net' +! stdout '^golang.org/x/net' +cmp stdout $WORK/net-deps.txt + + +# However, 'go mod' and 'go get' subcommands should report the original module +# dependencies, not the vendored packages. + +[!net] stop + +env GOPROXY= +go mod why -m golang.org/x/net +stdout '^# golang.org/x/net\nnet\ngolang.org/x/net' diff --git a/src/cmd/go/testdata/script/test_race_install_cgo.txt b/src/cmd/go/testdata/script/test_race_install_cgo.txt index 3f4eb90e3f..e1fe4f2ace 100644 --- a/src/cmd/go/testdata/script/test_race_install_cgo.txt +++ b/src/cmd/go/testdata/script/test_race_install_cgo.txt @@ -2,8 +2,6 @@ [!race] skip -[!darwin] ! stale cmd/cgo # The darwin builders are spuriously stale; see #33598. - env GOBIN=$WORK/bin go install m/mtime m/sametime diff --git a/src/cmd/go/testdata/script/toolexec.txt b/src/cmd/go/testdata/script/toolexec.txt index 4f26da6d26..bb86467942 100644 --- a/src/cmd/go/testdata/script/toolexec.txt +++ b/src/cmd/go/testdata/script/toolexec.txt @@ -3,6 +3,12 @@ # Build our simple toolexec program. go build ./cmd/mytool +# Use an ephemeral build cache so that our toolexec output is not cached +# for any stale standard-library dependencies. +# +# TODO(#27628): This should not be necessary. +env GOCACHE=$WORK/gocache + # Build the main package with our toolexec program. For each action, it will # print the tool's name and the TOOLEXEC_IMPORTPATH value. We expect to compile # each package once, and link the main package once. diff --git a/src/cmd/link/dwarf_test.go b/src/cmd/link/dwarf_test.go index 0419613cbe..3ca59bd47f 100644 --- a/src/cmd/link/dwarf_test.go +++ b/src/cmd/link/dwarf_test.go @@ -19,6 +19,36 @@ import ( "testing" ) +// TestMain allows this test binary to run as a -toolexec wrapper for the 'go' +// command. If LINK_TEST_TOOLEXEC is set, TestMain runs the binary as if it were +// cmd/link, and otherwise runs the requested tool as a subprocess. +// +// This allows the test to verify the behavior of the current contents of the +// cmd/link package even if the installed cmd/link binary is stale. +func TestMain(m *testing.M) { + if os.Getenv("LINK_TEST_TOOLEXEC") == "" { + // Not running as a -toolexec wrapper. Just run the tests. + os.Exit(m.Run()) + } + + if strings.TrimSuffix(filepath.Base(os.Args[1]), ".exe") == "link" { + // Running as a -toolexec linker, and the tool is cmd/link. + // Substitute this test binary for the linker. + os.Args = os.Args[1:] + main() + os.Exit(0) + } + + cmd := exec.Command(os.Args[1], os.Args[2:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + os.Exit(1) + } + os.Exit(0) +} + func testDWARF(t *testing.T, buildmode string, expectDWARF bool, env ...string) { testenv.MustHaveCGO(t) testenv.MustHaveGoBuild(t) @@ -29,17 +59,6 @@ func testDWARF(t *testing.T, buildmode string, expectDWARF bool, env ...string) t.Parallel() - out, err := exec.Command(testenv.GoToolPath(t), "list", "-f", "{{.Stale}}", "cmd/link").CombinedOutput() - if err != nil { - t.Fatalf("go list: %v\n%s", err, out) - } - if string(out) != "false\n" { - if strings.HasPrefix(testenv.Builder(), "darwin-") { - t.Skipf("cmd/link is spuriously stale on Darwin builders - see #33598") - } - t.Fatalf("cmd/link is stale - run go install cmd/link") - } - for _, prog := range []string{"testprog", "testprogcgo"} { prog := prog expectDWARF := expectDWARF @@ -48,11 +67,11 @@ func testDWARF(t *testing.T, buildmode string, expectDWARF bool, env ...string) if extld == "" { extld = "gcc" } + var err error expectDWARF, err = cmddwarf.IsDWARFEnabledOnAIXLd(extld) if err != nil { t.Fatal(err) } - } t.Run(prog, func(t *testing.T) { @@ -62,15 +81,14 @@ func testDWARF(t *testing.T, buildmode string, expectDWARF bool, env ...string) exe := filepath.Join(tmpDir, prog+".exe") dir := "../../runtime/testdata/" + prog - cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", exe) + cmd := exec.Command(testenv.GoToolPath(t), "build", "-toolexec", os.Args[0], "-o", exe) if buildmode != "" { cmd.Args = append(cmd.Args, "-buildmode", buildmode) } cmd.Args = append(cmd.Args, dir) - if env != nil { - cmd.Env = append(os.Environ(), env...) - cmd.Env = append(cmd.Env, "CGO_CFLAGS=") // ensure CGO_CFLAGS does not contain any flags. Issue #35459 - } + cmd.Env = append(os.Environ(), env...) + cmd.Env = append(cmd.Env, "CGO_CFLAGS=") // ensure CGO_CFLAGS does not contain any flags. Issue #35459 + cmd.Env = append(cmd.Env, "LINK_TEST_TOOLEXEC=1") out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("go build -o %v %v: %v\n%s", exe, dir, err, out) From 950fa11c4cb01a145bb07eeb167d90a1846061b3 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Fri, 21 May 2021 14:02:30 -0400 Subject: [PATCH 15/40] net/http/httputil: always remove hop-by-hop headers Previously, we'd fail to remove the Connection header from a request like this: Connection: Connection: x-header Fixes #46313 Fixes CVE-2021-33197 Change-Id: Ie3009e926ceecfa86dfa6bcc6fe14ff01086be7d Reviewed-on: https://go-review.googlesource.com/c/go/+/321929 Run-TryBot: Filippo Valsorda Reviewed-by: Katie Hockman Trust: Katie Hockman Trust: Filippo Valsorda TryBot-Result: Go Bot --- src/net/http/httputil/reverseproxy.go | 22 ++++---- src/net/http/httputil/reverseproxy_test.go | 63 +++++++++++++++++++++- 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go index 1432ee26d3..5d39955d62 100644 --- a/src/net/http/httputil/reverseproxy.go +++ b/src/net/http/httputil/reverseproxy.go @@ -253,22 +253,18 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // important is "Connection" because we want a persistent // connection, regardless of what the client sent to us. for _, h := range hopHeaders { - hv := outreq.Header.Get(h) - if hv == "" { - continue - } - if h == "Te" && hv == "trailers" { - // Issue 21096: tell backend applications that - // care about trailer support that we support - // trailers. (We do, but we don't go out of - // our way to advertise that unless the - // incoming client request thought it was - // worth mentioning) - continue - } outreq.Header.Del(h) } + // Issue 21096: tell backend applications that care about trailer support + // that we support trailers. (We do, but we don't go out of our way to + // advertise that unless the incoming client request thought it was worth + // mentioning.) Note that we look at req.Header, not outreq.Header, since + // the latter has passed through removeConnectionHeaders. + if httpguts.HeaderValuesContainsToken(req.Header["Te"], "trailers") { + outreq.Header.Set("Te", "trailers") + } + // After stripping all the hop-by-hop connection headers above, add back any // necessary for protocol upgrades, such as for websockets. if reqUpType != "" { diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go index 22720caf93..1898ed8b8a 100644 --- a/src/net/http/httputil/reverseproxy_test.go +++ b/src/net/http/httputil/reverseproxy_test.go @@ -91,8 +91,9 @@ func TestReverseProxy(t *testing.T) { getReq, _ := http.NewRequest("GET", frontend.URL, nil) getReq.Host = "some-name" - getReq.Header.Set("Connection", "close") - getReq.Header.Set("Te", "trailers") + getReq.Header.Set("Connection", "close, TE") + getReq.Header.Add("Te", "foo") + getReq.Header.Add("Te", "bar, trailers") getReq.Header.Set("Proxy-Connection", "should be deleted") getReq.Header.Set("Upgrade", "foo") getReq.Close = true @@ -236,6 +237,64 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { } } +func TestReverseProxyStripEmptyConnection(t *testing.T) { + // See Issue 46313. + const backendResponse = "I am the backend" + + // someConnHeader is some arbitrary header to be declared as a hop-by-hop header + // in the Request's Connection header. + const someConnHeader = "X-Some-Conn-Header" + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if c := r.Header.Values("Connection"); len(c) != 0 { + t.Errorf("handler got header %q = %v; want empty", "Connection", c) + } + if c := r.Header.Get(someConnHeader); c != "" { + t.Errorf("handler got header %q = %q; want empty", someConnHeader, c) + } + w.Header().Add("Connection", "") + w.Header().Add("Connection", someConnHeader) + w.Header().Set(someConnHeader, "should be deleted") + io.WriteString(w, backendResponse) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyHandler.ServeHTTP(w, r) + if c := r.Header.Get(someConnHeader); c != "should be deleted" { + t.Errorf("handler modified header %q = %q; want %q", someConnHeader, c, "should be deleted") + } + })) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Header.Add("Connection", "") + getReq.Header.Add("Connection", someConnHeader) + getReq.Header.Set(someConnHeader, "should be deleted") + res, err := frontend.Client().Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer res.Body.Close() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if got, want := string(bodyBytes), backendResponse; got != want { + t.Errorf("got body %q; want %q", got, want) + } + if c := res.Header.Get("Connection"); c != "" { + t.Errorf("handler got header %q = %q; want empty", "Connection", c) + } + if c := res.Header.Get(someConnHeader); c != "" { + t.Errorf("handler got header %q = %q; want empty", someConnHeader, c) + } +} + func TestXForwardedFor(t *testing.T) { const prevForwardedFor = "client ip" const backendResponse = "I am the backend" From fca7b8f3e690ec0562dd6ed609a8c7e6bef744c8 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Thu, 27 May 2021 15:14:18 +0000 Subject: [PATCH 16/40] Revert "net: verify results from Lookup* are valid domain names" This reverts commit c89f1224a544cde464fcb86e78ebb0cc97eedba2. Reason for revert: reverting so we can apply follow-up fixes and do a single cherry pick. Change-Id: I16c6283a0bcab056216f330fb98fa3b5f2b0780c Reviewed-on: https://go-review.googlesource.com/c/go/+/323129 Reviewed-by: Katie Hockman Reviewed-by: Filippo Valsorda Trust: Katie Hockman Run-TryBot: Katie Hockman TryBot-Result: Go Bot --- src/net/dnsclient_unix_test.go | 121 --------------------------------- src/net/lookup.go | 98 +++----------------------- 2 files changed, 8 insertions(+), 211 deletions(-) diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index 69a9b972f0..ec690a1c0c 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -1799,124 +1799,3 @@ func TestPTRandNonPTR(t *testing.T) { t.Errorf("names = %q; want %q", names, want) } } - -func TestCVE202133195(t *testing.T) { - fake := fakeDNSServer{ - rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { - r := dnsmessage.Message{ - Header: dnsmessage.Header{ - ID: q.Header.ID, - Response: true, - RCode: dnsmessage.RCodeSuccess, - RecursionAvailable: true, - }, - Questions: q.Questions, - } - switch q.Questions[0].Type { - case dnsmessage.TypeCNAME: - r.Answers = []dnsmessage.Resource{} - case dnsmessage.TypeA: // CNAME lookup uses a A/AAAA as a proxy - r.Answers = append(r.Answers, - dnsmessage.Resource{ - Header: dnsmessage.ResourceHeader{ - Name: dnsmessage.MustNewName(".golang.org."), - Type: dnsmessage.TypeA, - Class: dnsmessage.ClassINET, - Length: 4, - }, - Body: &dnsmessage.AResource{ - A: TestAddr, - }, - }, - ) - case dnsmessage.TypeSRV: - n := q.Questions[0].Name - if n.String() == "_hdr._tcp.golang.org." { - n = dnsmessage.MustNewName(".golang.org.") - } - r.Answers = append(r.Answers, - dnsmessage.Resource{ - Header: dnsmessage.ResourceHeader{ - Name: n, - Type: dnsmessage.TypeSRV, - Class: dnsmessage.ClassINET, - Length: 4, - }, - Body: &dnsmessage.SRVResource{ - Target: dnsmessage.MustNewName(".golang.org."), - }, - }, - ) - case dnsmessage.TypeMX: - r.Answers = append(r.Answers, - dnsmessage.Resource{ - Header: dnsmessage.ResourceHeader{ - Name: dnsmessage.MustNewName(".golang.org."), - Type: dnsmessage.TypeMX, - Class: dnsmessage.ClassINET, - Length: 4, - }, - Body: &dnsmessage.MXResource{ - MX: dnsmessage.MustNewName(".golang.org."), - }, - }, - ) - case dnsmessage.TypeNS: - r.Answers = append(r.Answers, - dnsmessage.Resource{ - Header: dnsmessage.ResourceHeader{ - Name: dnsmessage.MustNewName(".golang.org."), - Type: dnsmessage.TypeNS, - Class: dnsmessage.ClassINET, - Length: 4, - }, - Body: &dnsmessage.NSResource{ - NS: dnsmessage.MustNewName(".golang.org."), - }, - }, - ) - case dnsmessage.TypePTR: - r.Answers = append(r.Answers, - dnsmessage.Resource{ - Header: dnsmessage.ResourceHeader{ - Name: dnsmessage.MustNewName(".golang.org."), - Type: dnsmessage.TypePTR, - Class: dnsmessage.ClassINET, - Length: 4, - }, - Body: &dnsmessage.PTRResource{ - PTR: dnsmessage.MustNewName(".golang.org."), - }, - }, - ) - } - return r, nil - }, - } - r := Resolver{PreferGo: true, Dial: fake.DialContext} - - _, err := r.LookupCNAME(context.Background(), "golang.org") - if expected := "lookup golang.org: CNAME target is invalid"; err.Error() != expected { - t.Errorf("LookupCNAME returned unexpected error, got %q, want %q", err.Error(), expected) - } - _, _, err = r.LookupSRV(context.Background(), "target", "tcp", "golang.org") - if expected := "lookup golang.org: SRV target is invalid"; err.Error() != expected { - t.Errorf("LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) - } - _, _, err = r.LookupSRV(context.Background(), "hdr", "tcp", "golang.org") - if expected := "lookup golang.org: SRV header name is invalid"; err.Error() != expected { - t.Errorf("LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) - } - _, err = r.LookupMX(context.Background(), "golang.org") - if expected := "lookup golang.org: MX target is invalid"; err.Error() != expected { - t.Errorf("LookupMX returned unexpected error, got %q, want %q", err.Error(), expected) - } - _, err = r.LookupNS(context.Background(), "golang.org") - if expected := "lookup golang.org: NS target is invalid"; err.Error() != expected { - t.Errorf("LookupNS returned unexpected error, got %q, want %q", err.Error(), expected) - } - _, err = r.LookupAddr(context.Background(), "1.2.3.4") - if expected := "lookup 1.2.3.4: PTR target is invalid"; err.Error() != expected { - t.Errorf("LookupAddr returned unexpected error, got %q, want %q", err.Error(), expected) - } -} diff --git a/src/net/lookup.go b/src/net/lookup.go index 39d33796d5..03599503bd 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -396,9 +396,6 @@ func (r *Resolver) LookupPort(ctx context.Context, network, service string) (por // contain DNS "CNAME" records, as long as host resolves to // address records. // -// The returned canonical name is validated to be a properly -// formatted presentation-format domain name. -// // LookupCNAME uses context.Background internally; to specify the context, use // Resolver.LookupCNAME. func LookupCNAME(host string) (cname string, err error) { @@ -415,18 +412,8 @@ func LookupCNAME(host string) (cname string, err error) { // LookupCNAME does not return an error if host does not // contain DNS "CNAME" records, as long as host resolves to // address records. -// -// The returned canonical name is validated to be a properly -// formatted presentation-format domain name. -func (r *Resolver) LookupCNAME(ctx context.Context, host string) (string, error) { - cname, err := r.lookupCNAME(ctx, host) - if err != nil { - return "", err - } - if !isDomainName(cname) { - return "", &DNSError{Err: "CNAME target is invalid", Name: host} - } - return cname, nil +func (r *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, err error) { + return r.lookupCNAME(ctx, host) } // LookupSRV tries to resolve an SRV query of the given service, @@ -438,9 +425,6 @@ func (r *Resolver) LookupCNAME(ctx context.Context, host string) (string, error) // That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. -// -// The returned service names are validated to be properly -// formatted presentation-format domain names. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { return DefaultResolver.lookupSRV(context.Background(), service, proto, name) } @@ -454,33 +438,12 @@ func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err err // That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. -// -// The returned service names are validated to be properly -// formatted presentation-format domain names. -func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { - cname, addrs, err := r.lookupSRV(ctx, service, proto, name) - if err != nil { - return "", nil, err - } - if cname != "" && !isDomainName(cname) { - return "", nil, &DNSError{Err: "SRV header name is invalid", Name: name} - } - for _, addr := range addrs { - if addr == nil { - continue - } - if !isDomainName(addr.Target) { - return "", nil, &DNSError{Err: "SRV target is invalid", Name: name} - } - } - return cname, addrs, nil +func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { + return r.lookupSRV(ctx, service, proto, name) } // LookupMX returns the DNS MX records for the given domain name sorted by preference. // -// The returned mail server names are validated to be properly -// formatted presentation-format domain names. -// // LookupMX uses context.Background internally; to specify the context, use // Resolver.LookupMX. func LookupMX(name string) ([]*MX, error) { @@ -488,30 +451,12 @@ func LookupMX(name string) ([]*MX, error) { } // LookupMX returns the DNS MX records for the given domain name sorted by preference. -// -// The returned mail server names are validated to be properly -// formatted presentation-format domain names. func (r *Resolver) LookupMX(ctx context.Context, name string) ([]*MX, error) { - records, err := r.lookupMX(ctx, name) - if err != nil { - return nil, err - } - for _, mx := range records { - if mx == nil { - continue - } - if !isDomainName(mx.Host) { - return nil, &DNSError{Err: "MX target is invalid", Name: name} - } - } - return records, nil + return r.lookupMX(ctx, name) } // LookupNS returns the DNS NS records for the given domain name. // -// The returned name server names are validated to be properly -// formatted presentation-format domain names. -// // LookupNS uses context.Background internally; to specify the context, use // Resolver.LookupNS. func LookupNS(name string) ([]*NS, error) { @@ -519,23 +464,8 @@ func LookupNS(name string) ([]*NS, error) { } // LookupNS returns the DNS NS records for the given domain name. -// -// The returned name server names are validated to be properly -// formatted presentation-format domain names. func (r *Resolver) LookupNS(ctx context.Context, name string) ([]*NS, error) { - records, err := r.lookupNS(ctx, name) - if err != nil { - return nil, err - } - for _, ns := range records { - if ns == nil { - continue - } - if !isDomainName(ns.Host) { - return nil, &DNSError{Err: "NS target is invalid", Name: name} - } - } - return records, nil + return r.lookupNS(ctx, name) } // LookupTXT returns the DNS TXT records for the given domain name. @@ -565,18 +495,6 @@ func LookupAddr(addr string) (names []string, err error) { // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. -// -// The returned names are validated to be properly -// formatted presentation-format domain names. -func (r *Resolver) LookupAddr(ctx context.Context, addr string) ([]string, error) { - names, err := r.lookupAddr(ctx, addr) - if err != nil { - return nil, err - } - for _, name := range names { - if !isDomainName(name) { - return nil, &DNSError{Err: "PTR target is invalid", Name: addr} - } - } - return names, nil +func (r *Resolver) LookupAddr(ctx context.Context, addr string) (names []string, err error) { + return r.lookupAddr(ctx, addr) } From 6b8c94b6c524710bc3290546176a0da2f7c8c9db Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 27 May 2021 12:35:01 -0400 Subject: [PATCH 17/40] go/types: guard against check==nil in newNamed When importing generic named types, it is possible for Checker.newNamed to be called during type instantiation when the Checker is nil. In this case we should be able to safely skip this delayed expansion. Updates #45580 Change-Id: I75422100464d57eba24642c93e06e8b47d904fc3 Reviewed-on: https://go-review.googlesource.com/c/go/+/322974 Trust: Robert Findley Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/types/type.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/go/types/type.go b/src/go/types/type.go index 3fdb2365a0..2660ce4408 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -661,11 +661,7 @@ func NewNamed(obj *TypeName, underlying Type, methods []*Func) *Named { if _, ok := underlying.(*Named); ok { panic("types.NewNamed: underlying type must not be *Named") } - typ := &Named{obj: obj, orig: underlying, underlying: underlying, methods: methods} - if obj.typ == nil { - obj.typ = typ - } - return typ + return (*Checker)(nil).newNamed(obj, underlying, methods) } func (check *Checker) newNamed(obj *TypeName, underlying Type, methods []*Func) *Named { @@ -681,13 +677,15 @@ func (check *Checker) newNamed(obj *TypeName, underlying Type, methods []*Func) // // TODO(rFindley): clean this up so that under is the only function mutating // named types. - check.later(func() { - switch typ.under().(type) { - case *Named, *instance: - panic("internal error: unexpanded underlying type") - } - typ.check = nil - }) + if check != nil { + check.later(func() { + switch typ.under().(type) { + case *Named, *instance: + panic("internal error: unexpanded underlying type") + } + typ.check = nil + }) + } return typ } From db66e9e15d16cfdb555140b26a5f009fd0d23d0e Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Thu, 27 May 2021 10:11:42 -0400 Subject: [PATCH 18/40] cmd/link: accept Windows line-ending in TestTrampolineCgo Apparently C printf emits "\r\n" on Windows. Accept that. Change-Id: If87ba41435e3147d3892cfc3fe3a105b066ff0aa Reviewed-on: https://go-review.googlesource.com/c/go/+/322973 Trust: Cherry Mui Run-TryBot: Cherry Mui TryBot-Result: Go Bot Reviewed-by: Than McIntosh --- src/cmd/link/link_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/link/link_test.go b/src/cmd/link/link_test.go index 8805ff1f02..4d6bc76aca 100644 --- a/src/cmd/link/link_test.go +++ b/src/cmd/link/link_test.go @@ -698,7 +698,7 @@ func TestTrampolineCgo(t *testing.T) { if err != nil { t.Errorf("executable failed to run: %v\n%s", err, out) } - if string(out) != "hello\n" { + if string(out) != "hello\n" && string(out) != "hello\r\n" { t.Errorf("unexpected output:\n%s", out) } @@ -717,7 +717,7 @@ func TestTrampolineCgo(t *testing.T) { if err != nil { t.Errorf("executable failed to run: %v\n%s", err, out) } - if string(out) != "hello\n" { + if string(out) != "hello\n" && string(out) != "hello\r\n" { t.Errorf("unexpected output:\n%s", out) } } From 56af34f875f55485b4ebc521fe0c695dafb9bc23 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 25 May 2021 10:13:07 -0400 Subject: [PATCH 19/40] cmd/compile: place reg spills after OpArg{Int,Float}Reg ops Tweak the register allocator to maintain the invariant that OpArg{Int,Float}Reg values are placed together at the start of the entry block, before any other non-pseudo-op values. Without this change, when the register allocator adds spills we can wind up with an interleaving of OpArg*Reg and stores, which complicates debug location analysis. Updates #40724. Change-Id: Icf30dd814a9e25263ecbea2e48feb840a6e7f2bd Reviewed-on: https://go-review.googlesource.com/c/go/+/322630 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Cherry Mui --- src/cmd/compile/internal/ssa/regalloc.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/cmd/compile/internal/ssa/regalloc.go b/src/cmd/compile/internal/ssa/regalloc.go index c81d5574fe..3b90b8769c 100644 --- a/src/cmd/compile/internal/ssa/regalloc.go +++ b/src/cmd/compile/internal/ssa/regalloc.go @@ -1882,6 +1882,10 @@ func (s *regAllocState) placeSpills() { phiRegs[b.ID] = m } + mustBeFirst := func(op Op) bool { + return op.isLoweredGetClosurePtr() || op == OpPhi || op == OpArgIntReg || op == OpArgFloatReg + } + // Start maps block IDs to the list of spills // that go at the start of the block (but after any phis). start := map[ID][]*Value{} @@ -1971,7 +1975,7 @@ func (s *regAllocState) placeSpills() { // Put the spill in the best block we found. spill.Block = best spill.AddArg(bestArg) - if best == v.Block && v.Op != OpPhi { + if best == v.Block && !mustBeFirst(v.Op) { // Place immediately after v. after[v.ID] = append(after[v.ID], spill) } else { @@ -1983,15 +1987,15 @@ func (s *regAllocState) placeSpills() { // Insert spill instructions into the block schedules. var oldSched []*Value for _, b := range s.visitOrder { - nphi := 0 + nfirst := 0 for _, v := range b.Values { - if v.Op != OpPhi { + if !mustBeFirst(v.Op) { break } - nphi++ + nfirst++ } - oldSched = append(oldSched[:0], b.Values[nphi:]...) - b.Values = b.Values[:nphi] + oldSched = append(oldSched[:0], b.Values[nfirst:]...) + b.Values = b.Values[:nfirst] b.Values = append(b.Values, start[b.ID]...) for _, v := range oldSched { b.Values = append(b.Values, v) From 8bf5bf51738a198902e03bbec7e6ff220f4cb002 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Mon, 10 May 2021 07:31:57 -0400 Subject: [PATCH 20/40] cmd/compile: improve debug locations for partially live in-params During DWARF debug location generation, as a preamble to the main data flow analysis, examine the function entry block to look for in-params arriving in registers that are partially or completely dead, and insert new OpArg{Int,Float}Reg values for the dead or partially-dead pieces. In addition, add entries to the f.NamedValues table for incoming live register-resident params that don't already have entries. This helps create better/saner DWARF location expressions for params. Example: func foo(s string, used int, notused int) int { return len(s) + used } When optimization is complete for this function, the parameter "notused" is completely dead, meaning that there is no entry for it in the f.NamedValues table (which then means we don't emit a DWARF variable location expression for it in the function enty block). In addition, since only the length field of "s" is used, there is no DWARF location expression for the other component of "s", leading to degraded DWARF. There are still problems/issues with DWARF location generation, but this does improve things with respect to being able to print the values of incoming parameters when stopped in the debugger at the entry point of a function (when optimization is enabled). Updates #40724. Change-Id: I5bb5253648942f9fd33b081fe1a5a36208e75785 Reviewed-on: https://go-review.googlesource.com/c/go/+/322631 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Cherry Mui --- src/cmd/compile/internal/ssa/debug.go | 217 ++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/src/cmd/compile/internal/ssa/debug.go b/src/cmd/compile/internal/ssa/debug.go index a2c2a2d98e..eaa94975ec 100644 --- a/src/cmd/compile/internal/ssa/debug.go +++ b/src/cmd/compile/internal/ssa/debug.go @@ -7,10 +7,13 @@ package ssa import ( "cmd/compile/internal/abi" "cmd/compile/internal/ir" + "cmd/compile/internal/types" "cmd/internal/dwarf" "cmd/internal/obj" + "cmd/internal/src" "encoding/hex" "fmt" + "internal/buildcfg" "math/bits" "sort" "strings" @@ -335,6 +338,216 @@ func (s *debugState) stateString(state stateAtPC) string { return strings.Join(strs, "") } +// slotCanonicalizer is a table used to lookup and canonicalize +// LocalSlot's in a type insensitive way (e.g. taking into account the +// base name, offset, and width of the slot, but ignoring the slot +// type). +type slotCanonicalizer struct { + slmap map[slotKey]SlKeyIdx + slkeys []LocalSlot +} + +func newSlotCanonicalizer() *slotCanonicalizer { + return &slotCanonicalizer{ + slmap: make(map[slotKey]SlKeyIdx), + slkeys: []LocalSlot{LocalSlot{N: nil}}, + } +} + +type SlKeyIdx uint32 + +const noSlot = SlKeyIdx(0) + +// slotKey is a type-insensitive encapsulation of a LocalSlot; it +// is used to key a map within slotCanonicalizer. +type slotKey struct { + name *ir.Name + offset int64 + width int64 + splitOf SlKeyIdx // idx in slkeys slice in slotCanonicalizer + splitOffset int64 +} + +// lookup looks up a LocalSlot in the slot canonicalizer "sc", returning +// a canonical index for the slot, and adding it to the table if need +// be. Return value is the canonical slot index, and a boolean indicating +// whether the slot was found in the table already (TRUE => found). +func (sc *slotCanonicalizer) lookup(ls LocalSlot) (SlKeyIdx, bool) { + split := noSlot + if ls.SplitOf != nil { + split, _ = sc.lookup(*ls.SplitOf) + } + k := slotKey{ + name: ls.N, offset: ls.Off, width: ls.Type.Width, + splitOf: split, splitOffset: ls.SplitOffset, + } + if idx, ok := sc.slmap[k]; ok { + return idx, true + } + rv := SlKeyIdx(len(sc.slkeys)) + sc.slkeys = append(sc.slkeys, ls) + sc.slmap[k] = rv + return rv, false +} + +func (sc *slotCanonicalizer) canonSlot(idx SlKeyIdx) LocalSlot { + return sc.slkeys[idx] +} + +// PopulateABIInRegArgOps examines the entry block of the function +// and looks for incoming parameters that have missing or partial +// OpArg{Int,Float}Reg values, inserting additional values in +// cases where they are missing. Example: +// +// func foo(s string, used int, notused int) int { +// return len(s) + used +// } +// +// In the function above, the incoming parameter "used" is fully live, +// "notused" is not live, and "s" is partially live (only the length +// field of the string is used). At the point where debug value +// analysis runs, we might expect to see an entry block with: +// +// b1: +// v4 = ArgIntReg {s+8} [0] : BX +// v5 = ArgIntReg {used} [0] : CX +// +// While this is an accurate picture of the live incoming params, +// we also want to have debug locations for non-live params (or +// their non-live pieces), e.g. something like +// +// b1: +// v9 = ArgIntReg <*uint8> {s+0} [0] : AX +// v4 = ArgIntReg {s+8} [0] : BX +// v5 = ArgIntReg {used} [0] : CX +// v10 = ArgIntReg {unused} [0] : DI +// +// This function examines the live OpArg{Int,Float}Reg values and +// synthesizes new (dead) values for the non-live params or the +// non-live pieces of partially live params. +// +func PopulateABIInRegArgOps(f *Func) { + pri := f.ABISelf.ABIAnalyzeFuncType(f.Type.FuncType()) + + // When manufacturing new slots that correspond to splits of + // composite parameters, we want to avoid creating a new sub-slot + // that differs from some existing sub-slot only by type, since + // the debug location analysis will treat that slot as a separate + // entity. To achieve this, create a lookup table of existing + // slots that is type-insenstitive. + sc := newSlotCanonicalizer() + for _, sl := range f.Names { + sc.lookup(*sl) + } + + // Add slot -> value entry to f.NamedValues if not already present. + addToNV := func(v *Value, sl LocalSlot) { + values, ok := f.NamedValues[sl] + if !ok { + // Haven't seen this slot yet. + sla := f.localSlotAddr(sl) + f.Names = append(f.Names, sla) + } else { + for _, ev := range values { + if v == ev { + return + } + } + } + values = append(values, v) + f.NamedValues[sl] = values + } + + newValues := []*Value{} + + abiRegIndexToRegister := func(reg abi.RegIndex) int8 { + i := f.ABISelf.FloatIndexFor(reg) + if i >= 0 { // float PR + return f.Config.floatParamRegs[i] + } else { + return f.Config.intParamRegs[reg] + } + } + + // Helper to construct a new OpArg{Float,Int}Reg op value. + var pos src.XPos + if len(f.Entry.Values) != 0 { + pos = f.Entry.Values[0].Pos + } + synthesizeOpIntFloatArg := func(n *ir.Name, t *types.Type, reg abi.RegIndex, sl LocalSlot) *Value { + aux := &AuxNameOffset{n, sl.Off} + op, auxInt := ArgOpAndRegisterFor(reg, f.ABISelf) + v := f.newValueNoBlock(op, t, pos) + v.AuxInt = auxInt + v.Aux = aux + v.Args = nil + v.Block = f.Entry + newValues = append(newValues, v) + addToNV(v, sl) + f.setHome(v, &f.Config.registers[abiRegIndexToRegister(reg)]) + return v + } + + // Make a pass through the entry block looking for + // OpArg{Int,Float}Reg ops. Record the slots they use in a table + // ("sc"). We use a type-insensitive lookup for the slot table, + // since the type we get from the ABI analyzer won't always match + // what the compiler uses when creating OpArg{Int,Float}Reg ops. + for _, v := range f.Entry.Values { + if v.Op == OpArgIntReg || v.Op == OpArgFloatReg { + aux := v.Aux.(*AuxNameOffset) + sl := LocalSlot{N: aux.Name, Type: v.Type, Off: aux.Offset} + // install slot in lookup table + idx, _ := sc.lookup(sl) + // add to f.NamedValues if not already present + addToNV(v, sc.canonSlot(idx)) + } else if v.Op.IsCall() { + // if we hit a call, we've gone too far. + break + } + } + + // Now make a pass through the ABI in-params, looking for params + // or pieces of params that we didn't encounter in the loop above. + for _, inp := range pri.InParams() { + if !isNamedRegParam(inp) { + continue + } + n := inp.Name.(*ir.Name) + + // Param is spread across one or more registers. Walk through + // each piece to see whether we've seen an arg reg op for it. + types, offsets := inp.RegisterTypesAndOffsets() + for k, t := range types { + // Note: this recipe for creating a LocalSlot is designed + // to be compatible with the one used in expand_calls.go + // as opposed to decompose.go. The expand calls code just + // takes the base name and creates an offset into it, + // without using the SplitOf/SplitOffset fields. The code + // in decompose.go does the opposite -- it creates a + // LocalSlot object with "Off" set to zero, but with + // SplitOf pointing to a parent slot, and SplitOffset + // holding the offset into the parent object. + pieceSlot := LocalSlot{N: n, Type: t, Off: offsets[k]} + + // Look up this piece to see if we've seen a reg op + // for it. If not, create one. + _, found := sc.lookup(pieceSlot) + if !found { + // This slot doesn't appear in the map, meaning it + // corresponds to an in-param that is not live, or + // a portion of an in-param that is not live/used. + // Add a new dummy OpArg{Int,Float}Reg for it. + synthesizeOpIntFloatArg(n, t, inp.Registers[k], + pieceSlot) + } + } + } + + // Insert the new values into the head of the block. + f.Entry.Values = append(newValues, f.Entry.Values...) +} + // BuildFuncDebug returns debug information for f. // f must be fully processed, so that each Value is where it will be when // machine code is emitted. @@ -349,6 +562,10 @@ func BuildFuncDebug(ctxt *obj.Link, f *Func, loggingEnabled bool, stackOffset fu state.stackOffset = stackOffset state.ctxt = ctxt + if buildcfg.Experiment.RegabiArgs { + PopulateABIInRegArgOps(f) + } + if state.loggingEnabled { state.logf("Generating location lists for function %q\n", f.Name) } From cdcd02842da7c004efd023881e3719105209c908 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Thu, 27 May 2021 10:40:06 -0700 Subject: [PATCH 21/40] net: verify results from Lookup* are valid domain names For the methods LookupCNAME, LookupSRV, LookupMX, LookupNS, and LookupAddr check that the returned domain names are in fact valid DNS names using the existing isDomainName function. Thanks to Philipp Jeitner and Haya Shulman from Fraunhofer SIT for reporting this issue. Fixes #46241 Fixes CVE-2021-33195 Change-Id: I47a4f58c031cb752f732e88bbdae7f819f0af4f3 Reviewed-on: https://go-review.googlesource.com/c/go/+/323131 Trust: Roland Shoemaker Run-TryBot: Roland Shoemaker TryBot-Result: Go Bot Reviewed-by: Filippo Valsorda Reviewed-by: Katie Hockman --- src/net/dnsclient_unix_test.go | 157 +++++++++++++++++++++++++++++++++ src/net/lookup.go | 111 ++++++++++++++++++++--- 2 files changed, 255 insertions(+), 13 deletions(-) diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index ec690a1c0c..a718e75a72 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -1799,3 +1799,160 @@ func TestPTRandNonPTR(t *testing.T) { t.Errorf("names = %q; want %q", names, want) } } + +func TestCVE202133195(t *testing.T) { + fake := fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + r := dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.Header.ID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + RecursionAvailable: true, + }, + Questions: q.Questions, + } + switch q.Questions[0].Type { + case dnsmessage.TypeCNAME: + r.Answers = []dnsmessage.Resource{} + case dnsmessage.TypeA: // CNAME lookup uses a A/AAAA as a proxy + r.Answers = append(r.Answers, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName(".golang.org."), + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.AResource{ + A: TestAddr, + }, + }, + ) + case dnsmessage.TypeSRV: + n := q.Questions[0].Name + if n.String() == "_hdr._tcp.golang.org." { + n = dnsmessage.MustNewName(".golang.org.") + } + r.Answers = append(r.Answers, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: n, + Type: dnsmessage.TypeSRV, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.SRVResource{ + Target: dnsmessage.MustNewName(".golang.org."), + }, + }, + ) + case dnsmessage.TypeMX: + r.Answers = append(r.Answers, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName(".golang.org."), + Type: dnsmessage.TypeMX, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.MXResource{ + MX: dnsmessage.MustNewName(".golang.org."), + }, + }, + ) + case dnsmessage.TypeNS: + r.Answers = append(r.Answers, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName(".golang.org."), + Type: dnsmessage.TypeNS, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.NSResource{ + NS: dnsmessage.MustNewName(".golang.org."), + }, + }, + ) + case dnsmessage.TypePTR: + r.Answers = append(r.Answers, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName(".golang.org."), + Type: dnsmessage.TypePTR, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.PTRResource{ + PTR: dnsmessage.MustNewName(".golang.org."), + }, + }, + ) + } + return r, nil + }, + } + + r := Resolver{PreferGo: true, Dial: fake.DialContext} + // Change the default resolver to match our manipulated resolver + originalDefault := DefaultResolver + DefaultResolver = &r + defer func() { + DefaultResolver = originalDefault + }() + + _, err := r.LookupCNAME(context.Background(), "golang.org") + if expected := "lookup golang.org: CNAME target is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupCNAME returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, err = LookupCNAME("golang.org") + if expected := "lookup golang.org: CNAME target is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupCNAME returned unexpected error, got %q, want %q", err.Error(), expected) + } + + _, _, err = r.LookupSRV(context.Background(), "target", "tcp", "golang.org") + if expected := "lookup golang.org: SRV target is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, _, err = LookupSRV("target", "tcp", "golang.org") + if expected := "lookup golang.org: SRV target is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) + } + + _, _, err = r.LookupSRV(context.Background(), "hdr", "tcp", "golang.org") + if expected := "lookup golang.org: SRV header name is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, _, err = LookupSRV("hdr", "tcp", "golang.org") + if expected := "lookup golang.org: SRV header name is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupSRV returned unexpected error, got %q, want %q", err.Error(), expected) + } + + _, err = r.LookupMX(context.Background(), "golang.org") + if expected := "lookup golang.org: MX target is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupMX returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, err = LookupMX("golang.org") + if expected := "lookup golang.org: MX target is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupMX returned unexpected error, got %q, want %q", err.Error(), expected) + } + + _, err = r.LookupNS(context.Background(), "golang.org") + if expected := "lookup golang.org: NS target is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupNS returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, err = LookupNS("golang.org") + if expected := "lookup golang.org: NS target is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupNS returned unexpected error, got %q, want %q", err.Error(), expected) + } + + _, err = r.LookupAddr(context.Background(), "1.2.3.4") + if expected := "lookup 1.2.3.4: PTR target is invalid"; err == nil || err.Error() != expected { + t.Errorf("Resolver.LookupAddr returned unexpected error, got %q, want %q", err.Error(), expected) + } + _, err = LookupAddr("1.2.3.4") + if expected := "lookup 1.2.3.4: PTR target is invalid"; err == nil || err.Error() != expected { + t.Errorf("LookupAddr returned unexpected error, got %q, want %q", err.Error(), expected) + } +} diff --git a/src/net/lookup.go b/src/net/lookup.go index 03599503bd..02a4cdcd1e 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -396,10 +396,13 @@ func (r *Resolver) LookupPort(ctx context.Context, network, service string) (por // contain DNS "CNAME" records, as long as host resolves to // address records. // +// The returned canonical name is validated to be a properly +// formatted presentation-format domain name. +// // LookupCNAME uses context.Background internally; to specify the context, use // Resolver.LookupCNAME. func LookupCNAME(host string) (cname string, err error) { - return DefaultResolver.lookupCNAME(context.Background(), host) + return DefaultResolver.LookupCNAME(context.Background(), host) } // LookupCNAME returns the canonical name for the given host. @@ -412,8 +415,18 @@ func LookupCNAME(host string) (cname string, err error) { // LookupCNAME does not return an error if host does not // contain DNS "CNAME" records, as long as host resolves to // address records. -func (r *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, err error) { - return r.lookupCNAME(ctx, host) +// +// The returned canonical name is validated to be a properly +// formatted presentation-format domain name. +func (r *Resolver) LookupCNAME(ctx context.Context, host string) (string, error) { + cname, err := r.lookupCNAME(ctx, host) + if err != nil { + return "", err + } + if !isDomainName(cname) { + return "", &DNSError{Err: "CNAME target is invalid", Name: host} + } + return cname, nil } // LookupSRV tries to resolve an SRV query of the given service, @@ -425,8 +438,11 @@ func (r *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, // That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. +// +// The returned service names are validated to be properly +// formatted presentation-format domain names. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { - return DefaultResolver.lookupSRV(context.Background(), service, proto, name) + return DefaultResolver.LookupSRV(context.Background(), service, proto, name) } // LookupSRV tries to resolve an SRV query of the given service, @@ -438,34 +454,88 @@ func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err err // That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. -func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { - return r.lookupSRV(ctx, service, proto, name) +// +// The returned service names are validated to be properly +// formatted presentation-format domain names. +func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { + cname, addrs, err := r.lookupSRV(ctx, service, proto, name) + if err != nil { + return "", nil, err + } + if cname != "" && !isDomainName(cname) { + return "", nil, &DNSError{Err: "SRV header name is invalid", Name: name} + } + for _, addr := range addrs { + if addr == nil { + continue + } + if !isDomainName(addr.Target) { + return "", nil, &DNSError{Err: "SRV target is invalid", Name: name} + } + } + return cname, addrs, nil } // LookupMX returns the DNS MX records for the given domain name sorted by preference. // +// The returned mail server names are validated to be properly +// formatted presentation-format domain names. +// // LookupMX uses context.Background internally; to specify the context, use // Resolver.LookupMX. func LookupMX(name string) ([]*MX, error) { - return DefaultResolver.lookupMX(context.Background(), name) + return DefaultResolver.LookupMX(context.Background(), name) } // LookupMX returns the DNS MX records for the given domain name sorted by preference. +// +// The returned mail server names are validated to be properly +// formatted presentation-format domain names. func (r *Resolver) LookupMX(ctx context.Context, name string) ([]*MX, error) { - return r.lookupMX(ctx, name) + records, err := r.lookupMX(ctx, name) + if err != nil { + return nil, err + } + for _, mx := range records { + if mx == nil { + continue + } + if !isDomainName(mx.Host) { + return nil, &DNSError{Err: "MX target is invalid", Name: name} + } + } + return records, nil } // LookupNS returns the DNS NS records for the given domain name. // +// The returned name server names are validated to be properly +// formatted presentation-format domain names. +// // LookupNS uses context.Background internally; to specify the context, use // Resolver.LookupNS. func LookupNS(name string) ([]*NS, error) { - return DefaultResolver.lookupNS(context.Background(), name) + return DefaultResolver.LookupNS(context.Background(), name) } // LookupNS returns the DNS NS records for the given domain name. +// +// The returned name server names are validated to be properly +// formatted presentation-format domain names. func (r *Resolver) LookupNS(ctx context.Context, name string) ([]*NS, error) { - return r.lookupNS(ctx, name) + records, err := r.lookupNS(ctx, name) + if err != nil { + return nil, err + } + for _, ns := range records { + if ns == nil { + continue + } + if !isDomainName(ns.Host) { + return nil, &DNSError{Err: "NS target is invalid", Name: name} + } + } + return records, nil } // LookupTXT returns the DNS TXT records for the given domain name. @@ -484,17 +554,32 @@ func (r *Resolver) LookupTXT(ctx context.Context, name string) ([]string, error) // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. // +// The returned names are validated to be properly formatted presentation-format +// domain names. +// // When using the host C library resolver, at most one result will be // returned. To bypass the host resolver, use a custom Resolver. // // LookupAddr uses context.Background internally; to specify the context, use // Resolver.LookupAddr. func LookupAddr(addr string) (names []string, err error) { - return DefaultResolver.lookupAddr(context.Background(), addr) + return DefaultResolver.LookupAddr(context.Background(), addr) } // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. -func (r *Resolver) LookupAddr(ctx context.Context, addr string) (names []string, err error) { - return r.lookupAddr(ctx, addr) +// +// The returned names are validated to be properly formatted presentation-format +// domain names. +func (r *Resolver) LookupAddr(ctx context.Context, addr string) ([]string, error) { + names, err := r.lookupAddr(ctx, addr) + if err != nil { + return nil, err + } + for _, name := range names { + if !isDomainName(name) { + return nil, &DNSError{Err: "PTR target is invalid", Name: addr} + } + } + return names, nil } From 0ece95a0feafe151cadf1346464952bb5c95fab3 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 24 May 2021 16:47:45 -0400 Subject: [PATCH 22/40] cmd/go: don't let 'go mod download' save sums for inconsistent requirements 'go mod download' calls modload.LoadModFile early to find the main module path in order to validate arguments. LoadModFile may write go.mod and go.sum to fix formatting and add a go directive. This calls keepSums, which, in eager mode, loaded the complete module graph in order to find out what sums are needed to load the complete module graph. If go.mod requires a lower version of a module than will be selected later, keepSums causes the sum for that version's go.mod to be retained, even though it isn't needed later after a consistent go.mod is written. This CL fixes keepSums not to load the graph if it hasn't already been loaded (whether eager or lazy), addressing comments from CL 318629. For #45332 Change-Id: I20d4404004e4ad335450fd0fd753e7bc0060f702 Reviewed-on: https://go-review.googlesource.com/c/go/+/322369 Trust: Jay Conrod Run-TryBot: Bryan C. Mills Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot --- src/cmd/go/internal/modcmd/download.go | 16 ++++++++-------- src/cmd/go/internal/modload/init.go | 11 +++++------ src/cmd/go/testdata/script/mod_download.txt | 1 - 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 42b06dbc95..0e5af85237 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -138,14 +138,14 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { sem := make(chan token, runtime.GOMAXPROCS(0)) infos, infosErr := modload.ListModules(ctx, args, 0) if !haveExplicitArgs { - // 'go mod download' is sometimes run without arguments to pre-populate - // the module cache. It may fetch modules that aren't needed to build - // packages in the main mdoule. This is usually not intended, so don't save - // sums for downloaded modules (golang.org/issue/45332). - // TODO(golang.org/issue/45551): For now, save sums needed to load the - // build list (same as 1.15 behavior). In the future, report an error if - // go.mod or go.sum need to be updated after loading the build list. - modload.WriteGoMod(ctx) + // 'go mod download' is sometimes run without arguments to pre-populate the + // module cache. It may fetch modules that aren't needed to build packages + // in the main mdoule. This is usually not intended, so don't save sums for + // downloaded modules (golang.org/issue/45332). + // TODO(golang.org/issue/45551): For now, in ListModules, save sums needed + // to load the build list (same as 1.15 behavior). In the future, report an + // error if go.mod or go.sum need to be updated after loading the build + // list. modload.DisallowWriteGoMod() } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 86c0db3fe4..ea404b9f78 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -1122,12 +1122,11 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums } } - if rs.depth == lazy && rs.graph.Load() == nil { - // The main module is lazy and we haven't needed to load the module graph so - // far. Don't incur the cost of loading it now — since we haven't loaded the - // graph, we probably don't have any checksums to contribute to the distant - // parts of the graph anyway. Instead, just request sums for the roots that - // we know about. + if rs.graph.Load() == nil { + // The module graph was not loaded, possibly because the main module is lazy + // or possibly because we haven't needed to load the graph yet. + // Save sums for the root modules (or their replacements), but don't + // incur the cost of loading the graph just to find and retain the sums. for _, m := range rs.rootModules { r := resolveReplacement(m) keep[modkey(r)] = true diff --git a/src/cmd/go/testdata/script/mod_download.txt b/src/cmd/go/testdata/script/mod_download.txt index ad640b45de..c2b72b2a02 100644 --- a/src/cmd/go/testdata/script/mod_download.txt +++ b/src/cmd/go/testdata/script/mod_download.txt @@ -167,5 +167,4 @@ require ( -- update/go.sum.update -- golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= -rsc.io/sampler v1.2.1/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From ab2ef4aaa77dff91cd98ded88aeba0bf5b5b2e80 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 26 May 2021 18:06:26 -0700 Subject: [PATCH 23/40] doc/go1.17: document reflect changes For #44513. Fixes #46019. Change-Id: Ica84edd5703a4ccf343ff01e10483f8d51d1c79f Reviewed-on: https://go-review.googlesource.com/c/go/+/323069 Trust: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Go Bot Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor --- doc/go1.17.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index 8313c2bc57..d0a0c0f33f 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -500,11 +500,13 @@ Do not send CLs removing the interior tags from such phrases.

- TODO: https://golang.org/cl/281233: add VisibleFields function + The new VisibleFields function + returns all the visible fields in a struct type, including fields inside anonymous struct members.

- TODO: https://golang.org/cl/284136: panic if ArrayOf is called with negative length + The ArrayOf function now panics when + called with a negative length.

From 193d5141318d65cea310d995258288bd000d734c Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Thu, 27 May 2021 10:44:57 +0200 Subject: [PATCH 24/40] net/http: correct Client.Do doc about context cancelation The documentation of the Client.Do method and Get function incorrectly stated that, in case of context cancelation, the returned url.Error Timeout method returns true. Update the documentation to correctly match the implementation. See also CL 200798 that, due to an oversight, corrected only the documentation of the Client.Get method. Remove a TODO note added in CL 125575 (net/http: document that Client methods always return *url.Error), since it is no longer applicable after CL 200798 (net/http: fix and lock-in Client.Do docs on request cancelation). Fixes #46402 Change-Id: Ied2ee971ba22b61777762dbb19f16e08686634ca Reviewed-on: https://go-review.googlesource.com/c/go/+/323089 Reviewed-by: Damien Neil Trust: Michael Knyszek --- src/net/http/client.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/net/http/client.go b/src/net/http/client.go index 03c9155fbd..e0cabc9d4c 100644 --- a/src/net/http/client.go +++ b/src/net/http/client.go @@ -433,8 +433,7 @@ func basicAuth(username, password string) string { // An error is returned if there were too many redirects or if there // was an HTTP protocol error. A non-2xx response doesn't cause an // error. Any returned error will be of type *url.Error. The url.Error -// value's Timeout method will report true if request timed out or was -// canceled. +// value's Timeout method will report true if the request timed out. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. @@ -589,8 +588,7 @@ func urlErrorOp(method string) string { // standard library body types. // // Any returned error will be of type *url.Error. The url.Error -// value's Timeout method will report true if request timed out or was -// canceled. +// value's Timeout method will report true if the request timed out. func (c *Client) Do(req *Request) (*Response, error) { return c.do(req) } @@ -729,7 +727,6 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { reqBodyClosed = true if !deadline.IsZero() && didTimeout() { err = &httpError{ - // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancellation/ err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", timeout: true, } From 639acdc833bfd12b7edd43092d1b380d70cb2874 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 16:12:12 -0700 Subject: [PATCH 25/40] doc/go1.17: clarify that compress/lzw Reader and Writer types are new For #26535 For #44513 For #46005 Change-Id: I70d3711ab6451a61b526abb3da8e91243f637656 Reviewed-on: https://go-review.googlesource.com/c/go/+/323273 Trust: Ian Lance Taylor Reviewed-by: Joe Tsai --- doc/go1.17.html | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index d0a0c0f33f..b3485a0ca6 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -331,11 +331,16 @@ Do not send CLs removing the interior tags from such phrases.
compress/lzw

- The new - Reader.Reset - and - Writer.Reset - methods allow reuse of a Reader or Writer. + The NewReader + function is guaranteed to return a value of the new + type Reader, + and similarly NewWriter + is guaranteed to return a value of the new + type Writer. + These new types both implement a Reset method + (Reader.Reset, + Writer.Reset) + that allows reuse of the Reader or Writer.

From 3de3440fb9b2d7f8a14b33f96fcfcee8eb61ec55 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 27 May 2021 22:27:55 -0400 Subject: [PATCH 26/40] go/ast: remove FuncDecl.IsMethod for Go 1.17 The IsMethod method was added to FuncDecl in the process of working on support for type parameters, but is now only used in one place. It also didn't go through the proposal process. Remove it for 1.17. Also clean up a doc comment that mentioned type parameters. Fixes #46297 Change-Id: I432bdd626324f613baf059540b7c5436985b2b16 Reviewed-on: https://go-review.googlesource.com/c/go/+/323369 Trust: Robert Findley Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/ast/ast.go | 6 +----- src/go/types/resolver.go | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/go/ast/ast.go b/src/go/ast/ast.go index c87529ec77..337c87fd79 100644 --- a/src/go/ast/ast.go +++ b/src/go/ast/ast.go @@ -259,7 +259,7 @@ func (f *FieldList) End() token.Pos { return token.NoPos } -// NumFields returns the number of (type) parameters or struct fields represented by a FieldList. +// NumFields returns the number of parameters or struct fields represented by a FieldList. func (f *FieldList) NumFields() int { n := 0 if f != nil { @@ -973,10 +973,6 @@ type ( } ) -func (f *FuncDecl) IsMethod() bool { - return f.Recv.NumFields() != 0 -} - // Pos and End implementations for declaration nodes. func (d *BadDecl) Pos() token.Pos { return d.From } diff --git a/src/go/types/resolver.go b/src/go/types/resolver.go index f67fc65cd1..114647a2ff 100644 --- a/src/go/types/resolver.go +++ b/src/go/types/resolver.go @@ -383,7 +383,7 @@ func (check *Checker) collectObjects() { info := &declInfo{file: fileScope, fdecl: d.decl} name := d.decl.Name.Name obj := NewFunc(d.decl.Name.Pos(), pkg, name, nil) - if !d.decl.IsMethod() { + if d.decl.Recv.NumFields() == 0 { // regular function if d.decl.Recv != nil { check.error(d.decl.Recv, _BadRecv, "method is missing receiver") From ccd9784edf556673a340f3a8d55d9a8c64b95f59 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 16:16:37 -0700 Subject: [PATCH 27/40] doc/go1.17: document new debug/elf constant For #39677 For #44513 Change-Id: I8c4193fd4359b83e6739e7e30a3a42b5f21b0f1a Reviewed-on: https://go-review.googlesource.com/c/go/+/323275 Trust: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- doc/go1.17.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index b3485a0ca6..4e847708e5 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -373,6 +373,15 @@ Do not send CLs removing the interior tags from such phrases. +
debug/elf
+
+

+ The SHT_MIPS_ABIFLAGS + constant has been added. +

+
+
+
encoding/binary

From c295107708ad5fd26a78f9f1cb478f91aa7763e7 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 16:24:49 -0700 Subject: [PATCH 28/40] doc/go1.17: mention new encoding/csv/Reader.FieldPos method For #44221 For #44513 Change-Id: I2d2d1c55255f4411c11fd51f0f3ae726cbf4d136 Reviewed-on: https://go-review.googlesource.com/c/go/+/323349 Trust: Ian Lance Taylor Reviewed-by: roger peppe --- doc/go1.17.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 4e847708e5..9480b1205c 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -393,6 +393,18 @@ Do not send CLs removing the interior tags from such phrases.

+
encoding/csv
+
+

+ The new + Reader.FieldPos + method returns the line and column corresponding to the start of + a given field in the record most recently returned by + Read. +

+
+
+
flag

From 6f58088bd84daef583f30dcfdb7c2b9179bfff3a Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 17:19:12 -0700 Subject: [PATCH 29/40] doc/go1.17: document new go/build/BuildContext.ToolTags field For #44513 Change-Id: Ib21af742e574fcaa7e38bb437d42dbeed9d01f0b Reviewed-on: https://go-review.googlesource.com/c/go/+/323350 Trust: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- doc/go1.17.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 9480b1205c..65d8efdc1b 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -413,6 +413,17 @@ Do not send CLs removing the interior tags from such phrases.

+
go/build
+
+

+ The new + Context.ToolTags + field holds the build tags appropriate to the current Go + toolchain configuration. +

+
+
+
io/fs

From bbda92359289ae5992bbd8da9ce37932e531adda Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 17:30:08 -0700 Subject: [PATCH 30/40] doc/go1.17: mention new Windows SysProcAttr fields For #44011 For #44513 Change-Id: I512466f2e775e36098eb36ca7ef82333cd9e632a Reviewed-on: https://go-review.googlesource.com/c/go/+/323352 Trust: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- doc/go1.17.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 65d8efdc1b..3e9587dbf7 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -611,6 +611,15 @@ Do not send CLs removing the interior tags from such phrases. On Unix-like systems, the process group of a child process is now set with signals blocked. This avoids sending a SIGTTOU to the child when the parent is in a background process group.

+ +

+ The Windows version of + SysProcAttr + has two new fields. AdditionalInheritedHandles is + a list of additional handles to be inherited by the new child + process. ParentProcess permits specifying the + parent process of the new process. +

From 6624771c8346d69ef41526b1134c505bca399340 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 17:33:01 -0700 Subject: [PATCH 31/40] doc/go1.17: mention testing.[TB].Setenv methods For #41260 For #44513 Change-Id: I47ac0c751dafeb05abfe66fdf77938774164915f Reviewed-on: https://go-review.googlesource.com/c/go/+/323353 Trust: Ian Lance Taylor Reviewed-by: roger peppe --- doc/go1.17.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 3e9587dbf7..9d775e169a 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -628,6 +628,14 @@ Do not send CLs removing the interior tags from such phrases.

TODO: https://golang.org/cl/310033: add -shuffle=off|on|N to alter the execution order of tests and benchmarks

+ +

+ The new + T.Setenv + and B.Setenv + methods support setting an environment variable for the duration + of the test or benchmark. +

From 1419ca7cead4438c8c9f17d8901aeecd9c72f577 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 May 2021 17:25:43 -0700 Subject: [PATCH 32/40] doc/go1.17: mention new definitions of MSG_CMSG_CLOEXEC For #44513 Change-Id: I8c0070b116ee520a76726eb9d3dcbdd489a1fb1f Reviewed-on: https://go-review.googlesource.com/c/go/+/323351 Trust: Ian Lance Taylor Reviewed-by: Tobias Klauser Reviewed-by: Dmitri Shuralyov --- doc/go1.17.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 9d775e169a..3395c4e670 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -619,6 +619,11 @@ Do not send CLs removing the interior tags from such phrases. a list of additional handles to be inherited by the new child process. ParentProcess permits specifying the parent process of the new process. + +

+ The constant MSG_CMSG_CLOEXEC is now defined on + DragonFly and all OpenBSD systems (it was already defined on + some OpenBSD systems and all FreeBSD, NetBSD, and Linux systems).

From f6cc392d1ddd53a003c413fba4c1fc244ce2d85e Mon Sep 17 00:00:00 2001 From: Ariel Mashraki Date: Mon, 19 Apr 2021 22:26:47 +0300 Subject: [PATCH 33/40] doc/go1.17: document text/template/parse.SkipFuncCheck Documents the newly added mode that skips type checking functions as per CL 301493. Fixes #46025 For #34652 For #44513 For #38627 Change-Id: I56c4f65924702a931944796e39f43cfeb66abc8a Reviewed-on: https://go-review.googlesource.com/c/go/+/311569 Reviewed-by: Ian Lance Taylor Reviewed-by: Emmanuel Odeke Trust: Michael Knyszek --- doc/go1.17.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index 3395c4e670..3805a4c14e 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -647,7 +647,8 @@ Do not send CLs removing the interior tags from such phrases.
text/template/parse

- TODO: https://golang.org/cl/301493: add a mode to skip func-check on parsing + The new SkipFuncCheck Mode + value changes the template parser to not verify that functions are defined.

From 79bda650410c8617f0ae20dc552c6d5b8f8dcfc8 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 28 May 2021 17:32:15 -0700 Subject: [PATCH 34/40] doc/go1.17: mention time.Layout For #44513 Change-Id: Id4624e977654f7e8c489508a9dce98c9fab621a6 Reviewed-on: https://go-review.googlesource.com/c/go/+/323490 Trust: Ian Lance Taylor Reviewed-by: Rob Pike --- doc/go1.17.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/go1.17.html b/doc/go1.17.html index 3805a4c14e..02a58f8984 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -673,6 +673,11 @@ Do not send CLs removing the interior tags from such phrases.

TODO: https://golang.org/cl/300996: support "," as separator for fractional seconds

+ +

+ The new constant Layout + defines the reference time. +

From 1607c2817241bd141af9331a3e6c3148e5cd5d8b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 28 May 2021 11:58:05 -0400 Subject: [PATCH 35/40] go/types: unexport the GoVersion configuration option for Go 1.17 The GoVersion field was added to types.Config as part of the work on type parameters. Specifically, it was added to be consistent with cmd/compile/internal/types2, which requires such an option. This configuration option is useful, but is also non-trivial and did not go through the proposal process. Unexport it for Go 1.17; we can create a proposal to export it for Go 1.18. Fixes #46296 Change-Id: Id82d8a7096887dcfc404c4d6d8da9c761b316609 Reviewed-on: https://go-review.googlesource.com/c/go/+/323430 Trust: Robert Findley Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/types/api.go | 4 ++-- src/go/types/check.go | 4 ++-- src/go/types/check_test.go | 2 +- src/go/types/stdlib_test.go | 3 ++- src/go/types/types_test.go | 6 ++++++ 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/go/types/api.go b/src/go/types/api.go index ed62a785d6..8c0d9d22bf 100644 --- a/src/go/types/api.go +++ b/src/go/types/api.go @@ -101,12 +101,12 @@ type ImporterFrom interface { // A Config specifies the configuration for type checking. // The zero value for Config is a ready-to-use default configuration. type Config struct { - // GoVersion describes the accepted Go language version. The string + // goVersion describes the accepted Go language version. The string // must follow the format "go%d.%d" (e.g. "go1.12") or it must be // empty; an empty string indicates the latest language version. // If the format is invalid, invoking the type checker will cause a // panic. - GoVersion string + goVersion string // If IgnoreFuncBodies is set, function bodies are not // type-checked. diff --git a/src/go/types/check.go b/src/go/types/check.go index 25ea4906be..a923c3c612 100644 --- a/src/go/types/check.go +++ b/src/go/types/check.go @@ -179,9 +179,9 @@ func NewChecker(conf *Config, fset *token.FileSet, pkg *Package, info *Info) *Ch info = new(Info) } - version, err := parseGoVersion(conf.GoVersion) + version, err := parseGoVersion(conf.goVersion) if err != nil { - panic(fmt.Sprintf("invalid Go version %q (%v)", conf.GoVersion, err)) + panic(fmt.Sprintf("invalid Go version %q (%v)", conf.goVersion, err)) } return &Checker{ diff --git a/src/go/types/check_test.go b/src/go/types/check_test.go index c5dc93eade..9c71277264 100644 --- a/src/go/types/check_test.go +++ b/src/go/types/check_test.go @@ -240,7 +240,7 @@ func checkFiles(t *testing.T, sizes Sizes, goVersion string, filenames []string, // typecheck and collect typechecker errors var conf Config conf.Sizes = sizes - conf.GoVersion = goVersion + SetGoVersion(&conf, goVersion) // special case for importC.src if len(filenames) == 1 { diff --git a/src/go/types/stdlib_test.go b/src/go/types/stdlib_test.go index 3dea8dcf1e..503d0a6f44 100644 --- a/src/go/types/stdlib_test.go +++ b/src/go/types/stdlib_test.go @@ -134,7 +134,8 @@ func testTestDir(t *testing.T, path string, ignore ...string) { // parse and type-check file file, err := parser.ParseFile(fset, filename, nil, 0) if err == nil { - conf := Config{GoVersion: goVersion, Importer: stdLibImporter} + conf := Config{Importer: stdLibImporter} + SetGoVersion(&conf, goVersion) _, err = conf.Check(filename, fset, []*ast.File{file}, nil) } diff --git a/src/go/types/types_test.go b/src/go/types/types_test.go index fd9462c4a2..25cd996628 100644 --- a/src/go/types/types_test.go +++ b/src/go/types/types_test.go @@ -11,3 +11,9 @@ import "sync/atomic" // for tests where we may want to have a consistent // numbering for each individual test case. func ResetId() { atomic.StoreUint32(&lastId, 0) } + +// SetGoVersion sets the unexported goVersion field on config, so that tests +// which assert on behavior for older Go versions can set it. +func SetGoVersion(config *Config, goVersion string) { + config.goVersion = goVersion +} From 3b770f2ccb1fa6fecc22ea822a19447b10b70c5c Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Sat, 29 May 2021 22:14:12 -0400 Subject: [PATCH 36/40] go/types: don't declare 'comparable' when typeparams are disabled Fixes #46453 Change-Id: I92b9b1e43ec5182162b2eeeb667f1f548ea373a5 Reviewed-on: https://go-review.googlesource.com/c/go/+/323609 Trust: Robert Findley Run-TryBot: Robert Findley TryBot-Result: Go Bot Reviewed-by: Robert Griesemer --- src/go/types/check_test.go | 8 ++++++++ src/go/types/universe.go | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/go/types/check_test.go b/src/go/types/check_test.go index 9c71277264..6c3b630a1b 100644 --- a/src/go/types/check_test.go +++ b/src/go/types/check_test.go @@ -330,6 +330,14 @@ func TestIndexRepresentability(t *testing.T) { checkFiles(t, &StdSizes{4, 4}, "", []string{"index.go"}, [][]byte{[]byte(src)}, false) } +func TestIssue46453(t *testing.T) { + if typeparams.Enabled { + t.Skip("type params are enabled") + } + const src = "package p\ntype _ comparable // ERROR \"undeclared name: comparable\"" + checkFiles(t, nil, "", []string{"issue46453.go"}, [][]byte{[]byte(src)}, false) +} + func TestCheck(t *testing.T) { DefPredeclaredTestFuncs(); testDir(t, "check") } func TestExamples(t *testing.T) { testDir(t, "examples") } func TestFixedbugs(t *testing.T) { testDir(t, "fixedbugs") } diff --git a/src/go/types/universe.go b/src/go/types/universe.go index 7c211fa6f7..d7feb2c609 100644 --- a/src/go/types/universe.go +++ b/src/go/types/universe.go @@ -8,6 +8,7 @@ package types import ( "go/constant" + "go/internal/typeparams" "go/token" "strings" ) @@ -237,7 +238,9 @@ func init() { defPredeclaredConsts() defPredeclaredNil() defPredeclaredFuncs() - defPredeclaredComparable() + if typeparams.Enabled { + defPredeclaredComparable() + } universeIota = Universe.Lookup("iota").(*Const) universeByte = Universe.Lookup("byte").(*TypeName).typ.(*Basic) From 0b80cf11366f28ef5d0d8bae9c46813e96ffd071 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Wed, 26 May 2021 15:29:39 -0400 Subject: [PATCH 37/40] cmd/go: make 'go get' save sums for incidentally updated modules When 'go get' updates a module, it may update another module in the build list that provides a package in 'all' that wasn't loaded as part of the 'go get' command. If 'go get' doesn't add a sum for that module, builds may fail later. With this change, 'go get' will fetch a sum for the content of an updated module if we had a sum for the version before the update. 'go get' won't load the complete package graph, so there are still cases where the build may be broken, like when an updated (but not loaded) package imports a package from a new module. Fixes #44129 Change-Id: I62eba3df4137a3e84e2ca8d549c36eec3670f08c Reviewed-on: https://go-review.googlesource.com/c/go/+/322832 Trust: Jay Conrod Trust: Bryan C. Mills Run-TryBot: Jay Conrod Reviewed-by: Bryan C. Mills TryBot-Result: Go Bot --- src/cmd/go/internal/modget/get.go | 54 +++++++- .../script/mod_get_update_unrelated_sum.txt | 120 ++++++++++++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 src/cmd/go/testdata/script/mod_get_update_unrelated_sum.txt diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 563f1a988f..8eee723f89 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -38,6 +38,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/imports" "cmd/go/internal/load" + "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/par" "cmd/go/internal/search" @@ -1466,6 +1467,8 @@ func (r *resolver) chooseArbitrarily(cs pathSet) (isPackage bool, m module.Versi // checkPackageProblems reloads packages for the given patterns and reports // missing and ambiguous package errors. It also reports retractions and // deprecations for resolved modules and modules needed to build named packages. +// It also adds a sum for each updated module in the build list if we had one +// before and didn't get one while loading packages. // // We skip missing-package errors earlier in the process, since we want to // resolve pathSets ourselves, but at that point, we don't have enough context @@ -1593,9 +1596,52 @@ func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []strin }) } + // Load sums for updated modules that had sums before. When we update a + // module, we may update another module in the build list that provides a + // package in 'all' that wasn't loaded as part of this 'go get' command. + // If we don't add a sum for that module, builds may fail later. + // Note that an incidentally updated package could still import packages + // from unknown modules or from modules in the build list that we didn't + // need previously. We can't handle that case without loading 'all'. + sumErrs := make([]error, len(r.buildList)) + for i := range r.buildList { + i := i + m := r.buildList[i] + mActual := m + if mRepl := modload.Replacement(m); mRepl.Path != "" { + mActual = mRepl + } + old := module.Version{Path: m.Path, Version: r.initialVersion[m.Path]} + if old.Version == "" { + continue + } + oldActual := old + if oldRepl := modload.Replacement(old); oldRepl.Path != "" { + oldActual = oldRepl + } + if mActual == oldActual || mActual.Version == "" || !modfetch.HaveSum(oldActual) { + continue + } + r.work.Add(func() { + if _, err := modfetch.DownloadZip(ctx, mActual); err != nil { + verb := "upgraded" + if semver.Compare(m.Version, old.Version) < 0 { + verb = "downgraded" + } + replaced := "" + if mActual != m { + replaced = fmt.Sprintf(" (replaced by %s)", mActual) + } + err = fmt.Errorf("%s %s %s => %s%s: error finding sum for %s: %v", verb, m.Path, old.Version, m.Version, replaced, mActual, err) + sumErrs[i] = err + } + }) + } + <-r.work.Idle() - // Report deprecations, then retractions. + // Report deprecations, then retractions, then errors fetching sums. + // Only errors fetching sums are hard errors. for _, mm := range deprecations { if mm.message != "" { fmt.Fprintf(os.Stderr, "go: module %s is deprecated: %s\n", mm.m.Path, mm.message) @@ -1615,6 +1661,12 @@ func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []strin if retractPath != "" { fmt.Fprintf(os.Stderr, "go: to switch to the latest unretracted version, run:\n\tgo get %s@latest\n", retractPath) } + for _, err := range sumErrs { + if err != nil { + base.Errorf("go: %v", err) + } + } + base.ExitIfErrors() } // reportChanges logs version changes to os.Stderr. diff --git a/src/cmd/go/testdata/script/mod_get_update_unrelated_sum.txt b/src/cmd/go/testdata/script/mod_get_update_unrelated_sum.txt new file mode 100644 index 0000000000..0093c0eda0 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_get_update_unrelated_sum.txt @@ -0,0 +1,120 @@ +# Check that 'go get' adds sums for updated modules if we had sums before, +# even if we didn't load packages from them. +# Verifies #44129. + +env fmt='{{.ImportPath}}: {{if .Error}}{{.Error.Err}}{{else}}ok{{end}}' + +# Control case: before upgrading, we have the sums we need. +# go list -deps -e -f $fmt . +# stdout '^rsc.io/quote: ok$' +# ! stdout rsc.io/sampler # not imported by quote in this version +cp go.mod.orig go.mod +cp go.sum.orig go.sum +go mod tidy +cmp go.mod.orig go.mod +cmp go.sum.orig go.sum + + +# Upgrade a module. This also upgrades rsc.io/quote, and though we didn't load +# a package from it, we had the sum for its old version, so we need the +# sum for the new version, too. +go get -d example.com/upgrade@v0.0.2 +grep '^rsc.io/quote v1.5.2 ' go.sum + +# The upgrade still breaks the build because the new version of quote imports +# rsc.io/sampler, and we don't have its zip sum. +go list -deps -e -f $fmt +stdout 'rsc.io/quote: ok' +stdout 'rsc.io/sampler: missing go.sum entry for module providing package rsc.io/sampler' +cp go.mod.orig go.mod +cp go.sum.orig go.sum + + +# Replace the old version with a directory before upgrading. +# We didn't need a sum for it before (even though we had one), so we won't +# fetch a new sum. +go mod edit -replace rsc.io/quote@v1.0.0=./dummy +go get -d example.com/upgrade@v0.0.2 +! grep '^rsc.io/quote v1.5.2 ' go.sum +cp go.mod.orig go.mod +cp go.sum.orig go.sum + + +# Replace the new version with a directory before upgrading. +# We can't get a sum for a directory. +go mod edit -replace rsc.io/quote@v1.5.2=./dummy +go get -d example.com/upgrade@v0.0.2 +! grep '^rsc.io/quote v1.5.2 ' go.sum +cp go.mod.orig go.mod +cp go.sum.orig go.sum + + +# Replace the new version with a different version. +# We should get a sum for that version. +go mod edit -replace rsc.io/quote@v1.5.2=rsc.io/quote@v1.5.1 +go get -d example.com/upgrade@v0.0.2 +! grep '^rsc.io/quote v1.5.2 ' go.sum +grep '^rsc.io/quote v1.5.1 ' go.sum +cp go.mod.orig go.mod +cp go.sum.orig go.sum + + +# Delete the new version's zip (but not mod) from the cache and go offline. +# 'go get' should fail when fetching the zip. +rm $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip +env GOPROXY=off +! go get -d example.com/upgrade@v0.0.2 +stderr '^go: upgraded rsc.io/quote v1.0.0 => v1.5.2: error finding sum for rsc.io/quote@v1.5.2: module lookup disabled by GOPROXY=off$' + +-- go.mod.orig -- +module m + +go 1.16 + +require ( + example.com/upgrade v0.0.1 + rsc.io/quote v1.0.0 +) + +replace ( + example.com/upgrade v0.0.1 => ./upgrade1 + example.com/upgrade v0.0.2 => ./upgrade2 +) +-- go.sum.orig -- +rsc.io/quote v1.0.0 h1:kQ3IZQzPTiDJxSZI98YaWgxFEhlNdYASHvh+MplbViw= +rsc.io/quote v1.0.0/go.mod h1:v83Ri/njykPcgJltBc/gEkJTmjTsNgtO1Y7vyIK1CQA= +-- go.sum.want -- +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +rsc.io/quote v1.0.0 h1:kQ3IZQzPTiDJxSZI98YaWgxFEhlNdYASHvh+MplbViw= +rsc.io/quote v1.0.0/go.mod h1:v83Ri/njykPcgJltBc/gEkJTmjTsNgtO1Y7vyIK1CQA= +rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0= +rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +-- use.go -- +package use + +import ( + _ "example.com/upgrade" + _ "rsc.io/quote" +) +-- upgrade1/go.mod -- +module example.com/upgrade + +go 1.16 +-- upgrade1/upgrade.go -- +package upgrade +-- upgrade2/go.mod -- +module example.com/upgrade + +go 1.16 + +require rsc.io/quote v1.5.2 // indirect +-- upgrade2/upgrade.go -- +package upgrade +-- dummy/go.mod -- +module rsc.io/quote + +go 1.16 +-- dummy/quote.go -- +package quote + From 2e59cc5fb4e81fbd2ee9f662caa707c1138bf5ae Mon Sep 17 00:00:00 2001 From: Vitaly Zdanevich Date: Sat, 29 May 2021 13:49:51 +0000 Subject: [PATCH 38/40] cmd/go: add [-src] to documentation Change-Id: I554b5021386575af6ff44571a95bb31b38a0547f GitHub-Last-Rev: 20aaec3aa0baee6112fc2e4d72e83f78b72a44ea GitHub-Pull-Request: golang/go#45956 Reviewed-on: https://go-review.googlesource.com/c/go/+/317109 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Jay Conrod Trust: Bryan C. Mills --- src/cmd/go/alldocs.go | 2 +- src/cmd/go/internal/doc/doc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index bad2b7f16e..ab61017c4e 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -293,7 +293,7 @@ // // Usage: // -// go doc [-u] [-c] [package|[package.]symbol[.methodOrField]] +// go doc [doc flags] [package|[package.]symbol[.methodOrField]] // // Doc prints the documentation comments associated with the item identified by its // arguments (a package, const, func, type, var, method, or struct field) diff --git a/src/cmd/go/internal/doc/doc.go b/src/cmd/go/internal/doc/doc.go index 67f76e2256..8580a5dc4d 100644 --- a/src/cmd/go/internal/doc/doc.go +++ b/src/cmd/go/internal/doc/doc.go @@ -13,7 +13,7 @@ import ( var CmdDoc = &base.Command{ Run: runDoc, - UsageLine: "go doc [-u] [-c] [package|[package.]symbol[.methodOrField]]", + UsageLine: "go doc [doc flags] [package|[package.]symbol[.methodOrField]]", CustomFlags: true, Short: "show documentation for package or symbol", Long: ` From 2bec019fb5a7d379772c29be6e3487640d43e6fa Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Thu, 27 May 2021 10:13:00 -0400 Subject: [PATCH 39/40] doc/go1.17: add release notes for register ABI Also delete the TODO for the linker section. Updates #44513. Updates #40724. Change-Id: I4d62a907e8c3070831a052cdfe1e21648698df12 Reviewed-on: https://go-review.googlesource.com/c/go/+/323289 Trust: Cherry Mui Reviewed-by: Michael Knyszek Reviewed-by: Jeremy Faller --- doc/go1.17.html | 52 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/doc/go1.17.html b/doc/go1.17.html index 02a58f8984..ee498f7603 100644 --- a/doc/go1.17.html +++ b/doc/go1.17.html @@ -226,31 +226,57 @@ Do not send CLs removing the interior tags from such phrases.

Runtime

-

- TODO: https://golang.org/cl/304470: cmd/compile, runtime: add metadata for argument printing in traceback -

-

TODO: complete the Runtime section

Compiler

-

+

+ Go 1.17 implements a new way of passing function arguments and results using + registers instead of the stack. This work is enabled for Linux, MacOS, and + Windows on the 64-bit x86 architecture (the linux/amd64, + darwin/amd64, windows/amd64 ports). For a + representative set of Go packages and programs, benchmarking has shown + performance improvements of about 5%, and a typical reduction in binary size + of about 2%. +

+

+ This change does not affect the functionality of any safe Go code. It can affect + code outside the compatibility guidelines with + minimal impact. To maintain compatibility with existing assembly functions, + adapter functions converting between the new register-based calling convention + and the previous stack-based calling convention (also known as ABI wrappers) + are sometimes used. This is mostly invisible to users, except for assembly + functions that have their addresses taken in Go. Using reflect.ValueOf(fn).Pointer() + (or similar approaches such as via unsafe.Pointer) to get the address + of an assembly function will now return the address of the ABI wrapper. This is + mostly harmless, except for special-purpose assembly code (such as accessing + thread-local storage or requiring a special stack alignment). Assembly functions + called indirectly from Go via func values will now be made through + ABI wrappers, which may cause a very small performance overhead. Also, calling + Go functions from assembly may now go through ABI wrappers, with a very small + performance overhead. +

+ +

+ The format of stack traces from the runtime (printed when an uncaught panic + occurs, or when runtime.Stack is called) is improved. Previously, + the function arguments were printed as hexadecimal words based on the memory + layout. Now each argument in the source code is printed separately, separated + by commas. Aggregate-typed (struct, array, string, slice, interface, and complex) + arguments are delimited by curly braces. A caveat is that the value of an + argument that only lives in a register and is not stored to memory may be + inaccurate. Results (which were usually inaccurate) are no longer printed. +

+ +

Functions containing closures can now be inlined. One effect of this change is that a function with a closure may actually produce a distinct closure function for each place that the function is inlined. Hence, this change could reveal bugs where Go functions are compared (incorrectly) by pointer value. Go functions are by definition not comparable. - - TODO: complete the Compiler section, or delete if not needed -

- -

Linker

- -

- TODO: complete the Linker section, or delete if not needed

Core library

From 272552275f56345095b4ea7f404e5b317856cf07 Mon Sep 17 00:00:00 2001 From: OneOfOne Date: Fri, 21 May 2021 15:29:46 -0500 Subject: [PATCH 40/40] A+C: update name Change-Id: I1f88304858da5147bfd082c2fc2b7d24ed371cd8 Reviewed-on: https://go-review.googlesource.com/c/go/+/321955 Reviewed-by: Russ Cox Reviewed-by: Ian Lance Taylor --- AUTHORS | 2 +- CONTRIBUTORS | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 48ce71f4cc..95d3158d20 100644 --- a/AUTHORS +++ b/AUTHORS @@ -41,7 +41,7 @@ Aeneas Rekkas (arekkas) Afanasev Stanislav Agis Anastasopoulos Agniva De Sarker -Ahmed Wahed +Ahmed W. Mones Ahmet Soormally Ahmy Yulrizka Aiden Scandella diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 6c7262adb1..ee50a4c049 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -67,7 +67,7 @@ Aeneas Rekkas (arekkas) Afanasev Stanislav Agis Anastasopoulos Agniva De Sarker -Ahmed Wahed +Ahmed W. Mones Ahmet Alp Balkan Ahmet Soormally Ahmy Yulrizka