From 04bf36e97305197d09554739391f607afde1fd74 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Thu, 16 May 2024 21:24:53 +0100 Subject: [PATCH] encoding/binary: add Append, Encode and Decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a function which appends the binary representation of a value to the end of a slice. This allows users to encode values with zero allocations. Also add Encode and Decode functions which mimic unicode/utf8. goos: darwin goarch: arm64 pkg: encoding/binary cpu: Apple M1 Pro │ base.txt │ append.txt │ │ sec/op │ sec/op vs base │ ReadSlice1000Int32s-10 2.690µ ± 0% 2.532µ ± 3% -5.86% (p=0.002 n=6) ReadStruct-10 205.8n ± 0% 201.4n ± 1% -2.14% (p=0.002 n=6) WriteStruct-10 159.1n ± 0% 153.5n ± 0% -3.55% (p=0.002 n=6) WriteSlice1000Structs-10 129.8µ ± 0% 124.2µ ± 0% -4.34% (p=0.002 n=6) ReadSlice1000Structs-10 161.7µ ± 0% 160.3µ ± 0% -0.89% (p=0.002 n=6) ReadInts-10 156.8n ± 0% 161.6n ± 0% +3.09% (p=0.002 n=6) WriteInts-10 134.5n ± 0% 139.5n ± 2% +3.72% (p=0.002 n=6) WriteSlice1000Int32s-10 2.691µ ± 16% 2.551µ ± 4% -5.20% (p=0.002 n=6) PutUint16-10 0.6448n ± 4% 0.6212n ± 1% ~ (p=0.093 n=6) AppendUint16-10 1.414n ± 0% 1.424n ± 1% ~ (p=0.115 n=6) PutUint32-10 0.6210n ± 0% 0.6211n ± 0% ~ (p=0.833 n=6) AppendUint32-10 1.414n ± 0% 1.426n ± 1% +0.85% (p=0.017 n=6) PutUint64-10 0.6210n ± 0% 0.6394n ± 1% +2.95% (p=0.002 n=6) AppendUint64-10 1.414n ± 0% 1.427n ± 2% ~ (p=0.052 n=6) LittleEndianPutUint16-10 0.6239n ± 0% 0.6271n ± 1% ~ (p=0.063 n=6) LittleEndianAppendUint16-10 1.421n ± 0% 1.432n ± 1% +0.81% (p=0.002 n=6) LittleEndianPutUint32-10 0.6240n ± 0% 0.6240n ± 0% ~ (p=0.766 n=6) LittleEndianAppendUint32-10 1.422n ± 1% 1.425n ± 0% ~ (p=0.673 n=6) LittleEndianPutUint64-10 0.6242n ± 0% 0.6238n ± 0% -0.08% (p=0.030 n=6) LittleEndianAppendUint64-10 1.420n ± 0% 1.449n ± 1% +2.04% (p=0.002 n=6) ReadFloats-10 39.36n ± 0% 42.54n ± 1% +8.08% (p=0.002 n=6) WriteFloats-10 33.65n ± 0% 35.27n ± 1% +4.80% (p=0.002 n=6) ReadSlice1000Float32s-10 2.656µ ± 0% 2.526µ ± 1% -4.91% (p=0.002 n=6) WriteSlice1000Float32s-10 2.765µ ± 0% 2.857µ ± 3% +3.31% (p=0.002 n=6) ReadSlice1000Uint8s-10 129.1n ± 1% 130.4n ± 1% ~ (p=0.126 n=6) WriteSlice1000Uint8s-10 144.90n ± 3% 18.67n ± 2% -87.12% (p=0.002 n=6) PutUvarint32-10 12.11n ± 0% 12.12n ± 0% ~ (p=0.675 n=6) PutUvarint64-10 30.82n ± 0% 30.79n ± 1% ~ (p=0.658 n=6) AppendStruct-10 107.8n ± 0% AppendSlice1000Structs-10 119.0µ ± 0% AppendInts-10 55.29n ± 0% AppendSlice1000Int32s-10 2.211µ ± 1% geomean 33.07n 48.18n -7.03% Fixes #60023 Change-Id: Ife3f217b11d5f3eaa5a53fe8a7e877552f751f94 Reviewed-on: https://go-review.googlesource.com/c/go/+/579157 Reviewed-by: Keith Randall Auto-Submit: Austin Clements Reviewed-by: Ingo Oeser Reviewed-by: Austin Clements LUCI-TryBot-Result: Go LUCI --- api/next/60023.txt | 3 + .../99-minor/encoding/binary/60023.md | 3 + src/encoding/binary/binary.go | 504 +++++++++++------- src/encoding/binary/binary_test.go | 423 ++++++++++----- 4 files changed, 616 insertions(+), 317 deletions(-) create mode 100644 api/next/60023.txt create mode 100644 doc/next/6-stdlib/99-minor/encoding/binary/60023.md diff --git a/api/next/60023.txt b/api/next/60023.txt new file mode 100644 index 0000000000..4b5770830b --- /dev/null +++ b/api/next/60023.txt @@ -0,0 +1,3 @@ +pkg encoding/binary, func Encode([]uint8, ByteOrder, interface{}) (int, error) #60023 +pkg encoding/binary, func Decode([]uint8, ByteOrder, interface{}) (int, error) #60023 +pkg encoding/binary, func Append([]uint8, ByteOrder, interface{}) ([]uint8, error) #60023 diff --git a/doc/next/6-stdlib/99-minor/encoding/binary/60023.md b/doc/next/6-stdlib/99-minor/encoding/binary/60023.md new file mode 100644 index 0000000000..015bfc3803 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/encoding/binary/60023.md @@ -0,0 +1,3 @@ +The new [Encode] and [Decode] functions are byte slice equivalents +to [Read] and [Write]. +[Append] allows marshaling multiple data into the same byte slice. diff --git a/src/encoding/binary/binary.go b/src/encoding/binary/binary.go index 291e494dd4..55aa880ea5 100644 --- a/src/encoding/binary/binary.go +++ b/src/encoding/binary/binary.go @@ -26,9 +26,12 @@ import ( "io" "math" "reflect" + "slices" "sync" ) +var errBufferTooSmall = errors.New("buffer too small") + // A ByteOrder specifies how to convert byte slices into // 16-, 32-, or 64-bit unsigned integers. // @@ -236,80 +239,13 @@ func (nativeEndian) GoString() string { return "binary.NativeEndian" } // Read returns [io.ErrUnexpectedEOF]. func Read(r io.Reader, order ByteOrder, data any) error { // Fast path for basic types and slices. - if n := intDataSize(data); n != 0 { + if n, _ := intDataSize(data); n != 0 { bs := make([]byte, n) if _, err := io.ReadFull(r, bs); err != nil { return err } - switch data := data.(type) { - case *bool: - *data = bs[0] != 0 - case *int8: - *data = int8(bs[0]) - case *uint8: - *data = bs[0] - case *int16: - *data = int16(order.Uint16(bs)) - case *uint16: - *data = order.Uint16(bs) - case *int32: - *data = int32(order.Uint32(bs)) - case *uint32: - *data = order.Uint32(bs) - case *int64: - *data = int64(order.Uint64(bs)) - case *uint64: - *data = order.Uint64(bs) - case *float32: - *data = math.Float32frombits(order.Uint32(bs)) - case *float64: - *data = math.Float64frombits(order.Uint64(bs)) - case []bool: - for i, x := range bs { // Easier to loop over the input for 8-bit values. - data[i] = x != 0 - } - case []int8: - for i, x := range bs { - data[i] = int8(x) - } - case []uint8: - copy(data, bs) - case []int16: - for i := range data { - data[i] = int16(order.Uint16(bs[2*i:])) - } - case []uint16: - for i := range data { - data[i] = order.Uint16(bs[2*i:]) - } - case []int32: - for i := range data { - data[i] = int32(order.Uint32(bs[4*i:])) - } - case []uint32: - for i := range data { - data[i] = order.Uint32(bs[4*i:]) - } - case []int64: - for i := range data { - data[i] = int64(order.Uint64(bs[8*i:])) - } - case []uint64: - for i := range data { - data[i] = order.Uint64(bs[8*i:]) - } - case []float32: - for i := range data { - data[i] = math.Float32frombits(order.Uint32(bs[4*i:])) - } - case []float64: - for i := range data { - data[i] = math.Float64frombits(order.Uint64(bs[8*i:])) - } - default: - n = 0 // fast path doesn't apply - } - if n != 0 { + + if decodeFast(bs, order, data) { return nil } } @@ -327,6 +263,7 @@ func Read(r io.Reader, order ByteOrder, data any) error { if size < 0 { return errors.New("binary.Read: invalid type " + reflect.TypeOf(data).String()) } + d := &decoder{order: order, buf: make([]byte, size)} if _, err := io.ReadFull(r, d.buf); err != nil { return err @@ -335,6 +272,115 @@ func Read(r io.Reader, order ByteOrder, data any) error { return nil } +// Decode binary data from buf into data according to the given byte order. +// +// Returns an error if buf is too small, otherwise the number of +// bytes consumed from buf. +func Decode(buf []byte, order ByteOrder, data any) (int, error) { + if n, _ := intDataSize(data); n != 0 { + if len(buf) < n { + return 0, errBufferTooSmall + } + + if decodeFast(buf, order, data) { + return n, nil + } + } + + // Fallback to reflect-based decoding. + v := reflect.ValueOf(data) + size := -1 + switch v.Kind() { + case reflect.Pointer: + v = v.Elem() + size = dataSize(v) + case reflect.Slice: + size = dataSize(v) + } + if size < 0 { + return 0, errors.New("binary.Decode: invalid type " + reflect.TypeOf(data).String()) + } + + if len(buf) < size { + return 0, errBufferTooSmall + } + d := &decoder{order: order, buf: buf[:size]} + d.value(v) + return size, nil +} + +func decodeFast(bs []byte, order ByteOrder, data any) bool { + switch data := data.(type) { + case *bool: + *data = bs[0] != 0 + case *int8: + *data = int8(bs[0]) + case *uint8: + *data = bs[0] + case *int16: + *data = int16(order.Uint16(bs)) + case *uint16: + *data = order.Uint16(bs) + case *int32: + *data = int32(order.Uint32(bs)) + case *uint32: + *data = order.Uint32(bs) + case *int64: + *data = int64(order.Uint64(bs)) + case *uint64: + *data = order.Uint64(bs) + case *float32: + *data = math.Float32frombits(order.Uint32(bs)) + case *float64: + *data = math.Float64frombits(order.Uint64(bs)) + case []bool: + for i, x := range bs { // Easier to loop over the input for 8-bit values. + data[i] = x != 0 + } + case []int8: + for i, x := range bs { + data[i] = int8(x) + } + case []uint8: + copy(data, bs) + case []int16: + for i := range data { + data[i] = int16(order.Uint16(bs[2*i:])) + } + case []uint16: + for i := range data { + data[i] = order.Uint16(bs[2*i:]) + } + case []int32: + for i := range data { + data[i] = int32(order.Uint32(bs[4*i:])) + } + case []uint32: + for i := range data { + data[i] = order.Uint32(bs[4*i:]) + } + case []int64: + for i := range data { + data[i] = int64(order.Uint64(bs[8*i:])) + } + case []uint64: + for i := range data { + data[i] = order.Uint64(bs[8*i:]) + } + case []float32: + for i := range data { + data[i] = math.Float32frombits(order.Uint32(bs[4*i:])) + } + case []float64: + for i := range data { + data[i] = math.Float64frombits(order.Uint64(bs[8*i:])) + } + default: + return false + } + return true +} + // Write writes the binary representation of data into w. // Data must be a fixed-size value or a slice of fixed-size // values, or a pointer to such data. @@ -345,108 +391,12 @@ func Read(r io.Reader, order ByteOrder, data any) error { // with blank (_) field names. func Write(w io.Writer, order ByteOrder, data any) error { // Fast path for basic types and slices. - if n := intDataSize(data); n != 0 { - bs := make([]byte, n) - switch v := data.(type) { - case *bool: - if *v { - bs[0] = 1 - } else { - bs[0] = 0 - } - case bool: - if v { - bs[0] = 1 - } else { - bs[0] = 0 - } - case []bool: - for i, x := range v { - if x { - bs[i] = 1 - } else { - bs[i] = 0 - } - } - case *int8: - bs[0] = byte(*v) - case int8: - bs[0] = byte(v) - case []int8: - for i, x := range v { - bs[i] = byte(x) - } - case *uint8: - bs[0] = *v - case uint8: - bs[0] = v - case []uint8: - bs = v - case *int16: - order.PutUint16(bs, uint16(*v)) - case int16: - order.PutUint16(bs, uint16(v)) - case []int16: - for i, x := range v { - order.PutUint16(bs[2*i:], uint16(x)) - } - case *uint16: - order.PutUint16(bs, *v) - case uint16: - order.PutUint16(bs, v) - case []uint16: - for i, x := range v { - order.PutUint16(bs[2*i:], x) - } - case *int32: - order.PutUint32(bs, uint32(*v)) - case int32: - order.PutUint32(bs, uint32(v)) - case []int32: - for i, x := range v { - order.PutUint32(bs[4*i:], uint32(x)) - } - case *uint32: - order.PutUint32(bs, *v) - case uint32: - order.PutUint32(bs, v) - case []uint32: - for i, x := range v { - order.PutUint32(bs[4*i:], x) - } - case *int64: - order.PutUint64(bs, uint64(*v)) - case int64: - order.PutUint64(bs, uint64(v)) - case []int64: - for i, x := range v { - order.PutUint64(bs[8*i:], uint64(x)) - } - case *uint64: - order.PutUint64(bs, *v) - case uint64: - order.PutUint64(bs, v) - case []uint64: - for i, x := range v { - order.PutUint64(bs[8*i:], x) - } - case *float32: - order.PutUint32(bs, math.Float32bits(*v)) - case float32: - order.PutUint32(bs, math.Float32bits(v)) - case []float32: - for i, x := range v { - order.PutUint32(bs[4*i:], math.Float32bits(x)) - } - case *float64: - order.PutUint64(bs, math.Float64bits(*v)) - case float64: - order.PutUint64(bs, math.Float64bits(v)) - case []float64: - for i, x := range v { - order.PutUint64(bs[8*i:], math.Float64bits(x)) - } + if n, bs := intDataSize(data); n != 0 { + if bs == nil { + bs = make([]byte, n) + encodeFast(bs, order, data) } + _, err := w.Write(bs) return err } @@ -457,6 +407,7 @@ func Write(w io.Writer, order ByteOrder, data any) error { if size < 0 { return errors.New("binary.Write: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) } + buf := make([]byte, size) e := &encoder{order: order, buf: buf} e.value(v) @@ -464,6 +415,166 @@ func Write(w io.Writer, order ByteOrder, data any) error { return err } +// Encode the binary representation of data into buf according to the given byte order. +// +// Returns an error if the buffer is too short, otherwise the number of bytes +// written into buf. +func Encode(buf []byte, order ByteOrder, data any) (int, error) { + // Fast path for basic types and slices. + if n, _ := intDataSize(data); n != 0 { + if len(buf) < n { + return 0, errBufferTooSmall + } + + encodeFast(buf, order, data) + return n, nil + } + + // Fallback to reflect-based encoding. + v := reflect.Indirect(reflect.ValueOf(data)) + size := dataSize(v) + if size < 0 { + return 0, errors.New("binary.Encode: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) + } + + if len(buf) < size { + return 0, errBufferTooSmall + } + e := &encoder{order: order, buf: buf} + e.value(v) + return size, nil +} + +// Append the binary representation of data to buf. +// +// buf may be nil, in which case a new buffer will be allocated. +// See [Write] on which data are acceptable. +// +// Returns the (possibily extended) buffer containing data or an error. +func Append(buf []byte, order ByteOrder, data any) ([]byte, error) { + // Fast path for basic types and slices. + if n, _ := intDataSize(data); n != 0 { + buf, pos := ensure(buf, n) + encodeFast(pos, order, data) + return buf, nil + } + + // Fallback to reflect-based encoding. + v := reflect.Indirect(reflect.ValueOf(data)) + size := dataSize(v) + if size < 0 { + return nil, errors.New("binary.Append: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) + } + + buf, pos := ensure(buf, size) + e := &encoder{order: order, buf: pos} + e.value(v) + return buf, nil +} + +func encodeFast(bs []byte, order ByteOrder, data any) { + switch v := data.(type) { + case *bool: + if *v { + bs[0] = 1 + } else { + bs[0] = 0 + } + case bool: + if v { + bs[0] = 1 + } else { + bs[0] = 0 + } + case []bool: + for i, x := range v { + if x { + bs[i] = 1 + } else { + bs[i] = 0 + } + } + case *int8: + bs[0] = byte(*v) + case int8: + bs[0] = byte(v) + case []int8: + for i, x := range v { + bs[i] = byte(x) + } + case *uint8: + bs[0] = *v + case uint8: + bs[0] = v + case []uint8: + copy(bs, v) + case *int16: + order.PutUint16(bs, uint16(*v)) + case int16: + order.PutUint16(bs, uint16(v)) + case []int16: + for i, x := range v { + order.PutUint16(bs[2*i:], uint16(x)) + } + case *uint16: + order.PutUint16(bs, *v) + case uint16: + order.PutUint16(bs, v) + case []uint16: + for i, x := range v { + order.PutUint16(bs[2*i:], x) + } + case *int32: + order.PutUint32(bs, uint32(*v)) + case int32: + order.PutUint32(bs, uint32(v)) + case []int32: + for i, x := range v { + order.PutUint32(bs[4*i:], uint32(x)) + } + case *uint32: + order.PutUint32(bs, *v) + case uint32: + order.PutUint32(bs, v) + case []uint32: + for i, x := range v { + order.PutUint32(bs[4*i:], x) + } + case *int64: + order.PutUint64(bs, uint64(*v)) + case int64: + order.PutUint64(bs, uint64(v)) + case []int64: + for i, x := range v { + order.PutUint64(bs[8*i:], uint64(x)) + } + case *uint64: + order.PutUint64(bs, *v) + case uint64: + order.PutUint64(bs, v) + case []uint64: + for i, x := range v { + order.PutUint64(bs[8*i:], x) + } + case *float32: + order.PutUint32(bs, math.Float32bits(*v)) + case float32: + order.PutUint32(bs, math.Float32bits(v)) + case []float32: + for i, x := range v { + order.PutUint32(bs[4*i:], math.Float32bits(x)) + } + case *float64: + order.PutUint64(bs, math.Float64bits(*v)) + case float64: + order.PutUint64(bs, math.Float64bits(v)) + case []float64: + for i, x := range v { + order.PutUint64(bs[8*i:], math.Float64bits(x)) + } + } +} + // Size returns how many bytes [Write] would generate to encode the value v, which // must be a fixed-size value or a slice of fixed-size values, or a pointer to such data. // If v is neither of these, Size returns -1. @@ -766,44 +877,53 @@ func (e *encoder) skip(v reflect.Value) { e.offset += n } -// intDataSize returns the size of the data required to represent the data when encoded. -// It returns zero if the type cannot be implemented by the fast path in Read or Write. -func intDataSize(data any) int { +// intDataSize returns the size of the data required to represent the data when encoded, +// and optionally a byte slice containing the encoded data if no conversion is necessary. +// It returns zero, nil if the type cannot be implemented by the fast path in Read or Write. +func intDataSize(data any) (int, []byte) { switch data := data.(type) { case bool, int8, uint8, *bool, *int8, *uint8: - return 1 + return 1, nil case []bool: - return len(data) + return len(data), nil case []int8: - return len(data) + return len(data), nil case []uint8: - return len(data) + return len(data), data case int16, uint16, *int16, *uint16: - return 2 + return 2, nil case []int16: - return 2 * len(data) + return 2 * len(data), nil case []uint16: - return 2 * len(data) + return 2 * len(data), nil case int32, uint32, *int32, *uint32: - return 4 + return 4, nil case []int32: - return 4 * len(data) + return 4 * len(data), nil case []uint32: - return 4 * len(data) + return 4 * len(data), nil case int64, uint64, *int64, *uint64: - return 8 + return 8, nil case []int64: - return 8 * len(data) + return 8 * len(data), nil case []uint64: - return 8 * len(data) + return 8 * len(data), nil case float32, *float32: - return 4 + return 4, nil case float64, *float64: - return 8 + return 8, nil case []float32: - return 4 * len(data) + return 4 * len(data), nil case []float64: - return 8 * len(data) + return 8 * len(data), nil } - return 0 + return 0, nil +} + +// ensure grows buf to length len(buf) + n and returns the grown buffer +// and a slice starting at the original length of buf (that is, buf2[len(buf):]). +func ensure(buf []byte, n int) (buf2, pos []byte) { + l := len(buf) + buf = slices.Grow(buf, n)[:l+n] + return buf, buf[l:] } diff --git a/src/encoding/binary/binary_test.go b/src/encoding/binary/binary_test.go index 6cd0b92fa3..ca80c54c15 100644 --- a/src/encoding/binary/binary_test.go +++ b/src/encoding/binary/binary_test.go @@ -124,16 +124,81 @@ func checkResult(t *testing.T, dir string, order ByteOrder, err error, have, wan } } +var encoders = []struct { + name string + fn func(order ByteOrder, data any) ([]byte, error) +}{ + { + "Write", + func(order ByteOrder, data any) ([]byte, error) { + buf := new(bytes.Buffer) + err := Write(buf, order, data) + return buf.Bytes(), err + }, + }, + { + "Encode", + func(order ByteOrder, data any) ([]byte, error) { + size := Size(data) + + var buf []byte + if size > 0 { + buf = make([]byte, Size(data)) + } + + n, err := Encode(buf, order, data) + if err == nil && n != size { + return nil, fmt.Errorf("returned size %d instead of %d", n, size) + } + return buf, err + }, + }, { + "Append", + func(order ByteOrder, data any) ([]byte, error) { + return Append(nil, order, data) + }, + }, +} + +var decoders = []struct { + name string + fn func(order ByteOrder, data any, buf []byte) error +}{ + { + "Read", + func(order ByteOrder, data any, buf []byte) error { + return Read(bytes.NewReader(buf), order, data) + }, + }, + { + "Decode", + func(order ByteOrder, data any, buf []byte) error { + n, err := Decode(buf, order, data) + if err == nil && n != Size(data) { + return fmt.Errorf("returned size %d instead of %d", n, Size(data)) + } + return err + }, + }, +} + func testRead(t *testing.T, order ByteOrder, b []byte, s1 any) { - var s2 Struct - err := Read(bytes.NewReader(b), order, &s2) - checkResult(t, "Read", order, err, s2, s1) + for _, dec := range decoders { + t.Run(dec.name, func(t *testing.T) { + var s2 Struct + err := dec.fn(order, &s2, b) + checkResult(t, dec.name, order, err, s2, s1) + }) + } } func testWrite(t *testing.T, order ByteOrder, b []byte, s1 any) { - buf := new(bytes.Buffer) - err := Write(buf, order, s1) - checkResult(t, "Write", order, err, buf.Bytes(), b) + for _, enc := range encoders { + t.Run(enc.name, func(t *testing.T) { + buf, err := enc.fn(order, s1) + checkResult(t, enc.name, order, err, buf, b) + }) + } } func TestLittleEndianRead(t *testing.T) { testRead(t, LittleEndian, little, s) } @@ -145,34 +210,49 @@ func TestBigEndianWrite(t *testing.T) { testWrite(t, BigEndian, big, s) } func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) } func TestReadSlice(t *testing.T) { - slice := make([]int32, 2) - err := Read(bytes.NewReader(src), BigEndian, slice) - checkResult(t, "ReadSlice", BigEndian, err, slice, res) + t.Run("Read", func(t *testing.T) { + slice := make([]int32, 2) + err := Read(bytes.NewReader(src), BigEndian, slice) + checkResult(t, "ReadSlice", BigEndian, err, slice, res) + }) + + t.Run("Decode", func(t *testing.T) { + slice := make([]int32, 2) + _, err := Decode(src, BigEndian, slice) + checkResult(t, "ReadSlice", BigEndian, err, slice, res) + }) } func TestWriteSlice(t *testing.T) { - buf := new(bytes.Buffer) - err := Write(buf, BigEndian, res) - checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) + testWrite(t, BigEndian, src, res) } func TestReadBool(t *testing.T) { - var res bool - var err error - err = Read(bytes.NewReader([]byte{0}), BigEndian, &res) - checkResult(t, "ReadBool", BigEndian, err, res, false) - res = false - err = Read(bytes.NewReader([]byte{1}), BigEndian, &res) - checkResult(t, "ReadBool", BigEndian, err, res, true) - res = false - err = Read(bytes.NewReader([]byte{2}), BigEndian, &res) - checkResult(t, "ReadBool", BigEndian, err, res, true) + for _, dec := range decoders { + t.Run(dec.name, func(t *testing.T) { + var res bool + var err error + err = dec.fn(BigEndian, &res, []byte{0}) + checkResult(t, dec.name, BigEndian, err, res, false) + res = false + err = dec.fn(BigEndian, &res, []byte{1}) + checkResult(t, dec.name, BigEndian, err, res, true) + res = false + err = dec.fn(BigEndian, &res, []byte{2}) + checkResult(t, dec.name, BigEndian, err, res, true) + }) + } + } func TestReadBoolSlice(t *testing.T) { - slice := make([]bool, 4) - err := Read(bytes.NewReader([]byte{0, 1, 2, 255}), BigEndian, slice) - checkResult(t, "ReadBoolSlice", BigEndian, err, slice, []bool{false, true, true, true}) + for _, dec := range decoders { + t.Run(dec.name, func(t *testing.T) { + slice := make([]bool, 4) + err := dec.fn(BigEndian, slice, []byte{0, 1, 2, 255}) + checkResult(t, dec.name, BigEndian, err, slice, []bool{false, true, true, true}) + }) + } } // Addresses of arrays are easier to manipulate with reflection than are slices. @@ -188,57 +268,67 @@ var intArrays = []any{ } func TestSliceRoundTrip(t *testing.T) { - buf := new(bytes.Buffer) - for _, array := range intArrays { - src := reflect.ValueOf(array).Elem() - unsigned := false - switch src.Index(0).Kind() { - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - unsigned = true - } - for i := 0; i < src.Len(); i++ { - if unsigned { - src.Index(i).SetUint(uint64(i * 0x07654321)) - } else { - src.Index(i).SetInt(int64(i * 0x07654321)) - } - } - buf.Reset() - srcSlice := src.Slice(0, src.Len()) - err := Write(buf, BigEndian, srcSlice.Interface()) - if err != nil { - t.Fatal(err) - } - dst := reflect.New(src.Type()).Elem() - dstSlice := dst.Slice(0, dst.Len()) - err = Read(buf, BigEndian, dstSlice.Interface()) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(src.Interface(), dst.Interface()) { - t.Fatal(src) + for _, enc := range encoders { + for _, dec := range decoders { + t.Run(fmt.Sprintf("%s,%s", enc.name, dec.name), func(t *testing.T) { + for _, array := range intArrays { + src := reflect.ValueOf(array).Elem() + t.Run(src.Index(0).Type().Name(), func(t *testing.T) { + unsigned := false + switch src.Index(0).Kind() { + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + unsigned = true + } + for i := 0; i < src.Len(); i++ { + if unsigned { + src.Index(i).SetUint(uint64(i * 0x07654321)) + } else { + src.Index(i).SetInt(int64(i * 0x07654321)) + } + } + srcSlice := src.Slice(0, src.Len()) + buf, err := enc.fn(BigEndian, srcSlice.Interface()) + if err != nil { + t.Fatal(err) + } + dst := reflect.New(src.Type()).Elem() + dstSlice := dst.Slice(0, dst.Len()) + err = dec.fn(BigEndian, dstSlice.Interface(), buf) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(src.Interface(), dst.Interface()) { + t.Log(dst) + t.Fatal(src) + } + }) + } + }) } } } func TestWriteT(t *testing.T) { - buf := new(bytes.Buffer) - ts := T{} - if err := Write(buf, BigEndian, ts); err == nil { - t.Errorf("WriteT: have err == nil, want non-nil") - } + for _, enc := range encoders { + t.Run(enc.name, func(t *testing.T) { + ts := T{} + if _, err := enc.fn(BigEndian, ts); err == nil { + t.Errorf("WriteT: have err == nil, want non-nil") + } - tv := reflect.Indirect(reflect.ValueOf(ts)) - for i, n := 0, tv.NumField(); i < n; i++ { - typ := tv.Field(i).Type().String() - if typ == "[4]int" { - typ = "int" // the problem is int, not the [4] - } - if err := Write(buf, BigEndian, tv.Field(i).Interface()); err == nil { - t.Errorf("WriteT.%v: have err == nil, want non-nil", tv.Field(i).Type()) - } else if !strings.Contains(err.Error(), typ) { - t.Errorf("WriteT: have err == %q, want it to mention %s", err, typ) - } + tv := reflect.Indirect(reflect.ValueOf(ts)) + for i, n := 0, tv.NumField(); i < n; i++ { + typ := tv.Field(i).Type().String() + if typ == "[4]int" { + typ = "int" // the problem is int, not the [4] + } + if _, err := enc.fn(BigEndian, tv.Field(i).Interface()); err == nil { + t.Errorf("WriteT.%v: have err == nil, want non-nil", tv.Field(i).Type()) + } else if !strings.Contains(err.Error(), typ) { + t.Errorf("WriteT: have err == %q, want it to mention %s", err, typ) + } + } + }) } } @@ -267,35 +357,40 @@ type BlankFieldsProbe struct { } func TestBlankFields(t *testing.T) { - buf := new(bytes.Buffer) - b1 := BlankFields{A: 1234567890, B: 2.718281828, C: 42} - if err := Write(buf, LittleEndian, &b1); err != nil { - t.Error(err) - } + for _, enc := range encoders { + t.Run(enc.name, func(t *testing.T) { + b1 := BlankFields{A: 1234567890, B: 2.718281828, C: 42} + buf, err := enc.fn(LittleEndian, &b1) + if err != nil { + t.Error(err) + } - // zero values must have been written for blank fields - var p BlankFieldsProbe - if err := Read(buf, LittleEndian, &p); err != nil { - t.Error(err) - } + // zero values must have been written for blank fields + var p BlankFieldsProbe + if err := Read(bytes.NewReader(buf), LittleEndian, &p); err != nil { + t.Error(err) + } - // quick test: only check first value of slices - if p.P0 != 0 || p.P1[0] != 0 || p.P2[0] != 0 || p.P3.F[0] != 0 { - t.Errorf("non-zero values for originally blank fields: %#v", p) - } + // quick test: only check first value of slices + if p.P0 != 0 || p.P1[0] != 0 || p.P2[0] != 0 || p.P3.F[0] != 0 { + t.Errorf("non-zero values for originally blank fields: %#v", p) + } - // write p and see if we can probe only some fields - if err := Write(buf, LittleEndian, &p); err != nil { - t.Error(err) - } + // write p and see if we can probe only some fields + buf, err = enc.fn(LittleEndian, &p) + if err != nil { + t.Error(err) + } - // read should ignore blank fields in b2 - var b2 BlankFields - if err := Read(buf, LittleEndian, &b2); err != nil { - t.Error(err) - } - if b1.A != b2.A || b1.B != b2.B || b1.C != b2.C { - t.Errorf("%#v != %#v", b1, b2) + // read should ignore blank fields in b2 + var b2 BlankFields + if err := Read(bytes.NewReader(buf), LittleEndian, &b2); err != nil { + t.Error(err) + } + if b1.A != b2.A || b1.B != b2.B || b1.C != b2.C { + t.Errorf("%#v != %#v", b1, b2) + } + }) } } @@ -386,33 +481,41 @@ func TestUnexportedRead(t *testing.T) { t.Fatal(err) } - defer func() { - if recover() == nil { - t.Fatal("did not panic") - } - }() - var u2 Unexported - Read(&buf, LittleEndian, &u2) + for _, dec := range decoders { + t.Run(dec.name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("did not panic") + } + }() + var u2 Unexported + dec.fn(LittleEndian, &u2, buf.Bytes()) + }) + } + } func TestReadErrorMsg(t *testing.T) { - var buf bytes.Buffer - read := func(data any) { - err := Read(&buf, LittleEndian, data) - want := "binary.Read: invalid type " + reflect.TypeOf(data).String() - if err == nil { - t.Errorf("%T: got no error; want %q", data, want) - return - } - if got := err.Error(); got != want { - t.Errorf("%T: got %q; want %q", data, got, want) - } + for _, dec := range decoders { + t.Run(dec.name, func(t *testing.T) { + read := func(data any) { + err := dec.fn(LittleEndian, data, nil) + want := fmt.Sprintf("binary.%s: invalid type %s", dec.name, reflect.TypeOf(data).String()) + if err == nil { + t.Errorf("%T: got no error; want %q", data, want) + return + } + if got := err.Error(); got != want { + t.Errorf("%T: got %q; want %q", data, got, want) + } + } + read(0) + s := new(struct{}) + read(&s) + p := &s + read(&p) + }) } - read(0) - s := new(struct{}) - read(&s) - p := &s - read(&p) } func TestReadTruncated(t *testing.T) { @@ -573,14 +676,31 @@ func TestNoFixedSize(t *testing.T) { Height: 177.8, } - buf := new(bytes.Buffer) - err := Write(buf, LittleEndian, &person) - if err == nil { - t.Fatal("binary.Write: unexpected success as size of type *binary.Person is not fixed") + for _, enc := range encoders { + t.Run(enc.name, func(t *testing.T) { + _, err := enc.fn(LittleEndian, &person) + if err == nil { + t.Fatalf("binary.%s: unexpected success as size of type *binary.Person is not fixed", enc.name) + } + errs := fmt.Sprintf("binary.%s: some values are not fixed-sized in type *binary.Person", enc.name) + if err.Error() != errs { + t.Fatalf("got %q, want %q", err, errs) + } + }) } - errs := "binary.Write: some values are not fixed-sized in type *binary.Person" - if err.Error() != errs { - t.Fatalf("got %q, want %q", err, errs) +} + +func TestAppendAllocs(t *testing.T) { + buf := make([]byte, 0, Size(&s)) + var err error + allocs := testing.AllocsPerRun(1, func() { + _, err = Append(buf, LittleEndian, &s) + }) + if err != nil { + t.Fatal("Append failed:", err) + } + if allocs != 0 { + t.Fatalf("Append allocated %v times instead of not allocating at all", allocs) } } @@ -631,6 +751,16 @@ func BenchmarkWriteStruct(b *testing.B) { } } +func BenchmarkAppendStruct(b *testing.B) { + buf := make([]byte, 0, Size(&s)) + b.SetBytes(int64(cap(buf))) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + Encode(buf, BigEndian, &s) + } +} + func BenchmarkWriteSlice1000Structs(b *testing.B) { slice := make([]Struct, 1000) buf := new(bytes.Buffer) @@ -644,6 +774,17 @@ func BenchmarkWriteSlice1000Structs(b *testing.B) { b.StopTimer() } +func BenchmarkAppendSlice1000Structs(b *testing.B) { + slice := make([]Struct, 1000) + buf := make([]byte, 0, Size(slice)) + b.SetBytes(int64(cap(buf))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Append(buf, BigEndian, slice) + } + b.StopTimer() +} + func BenchmarkReadSlice1000Structs(b *testing.B) { bsr := &byteSliceReader{} slice := make([]Struct, 1000) @@ -709,6 +850,27 @@ func BenchmarkWriteInts(b *testing.B) { } } +func BenchmarkAppendInts(b *testing.B) { + buf := make([]byte, 0, 256) + b.SetBytes(2 * (1 + 2 + 4 + 8)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf = buf[:0] + buf, _ = Append(buf, BigEndian, s.Int8) + buf, _ = Append(buf, BigEndian, s.Int16) + buf, _ = Append(buf, BigEndian, s.Int32) + buf, _ = Append(buf, BigEndian, s.Int64) + buf, _ = Append(buf, BigEndian, s.Uint8) + buf, _ = Append(buf, BigEndian, s.Uint16) + buf, _ = Append(buf, BigEndian, s.Uint32) + buf, _ = Append(buf, BigEndian, s.Uint64) + } + b.StopTimer() + if b.N > 0 && !bytes.Equal(buf, big[:30]) { + b.Fatalf("first half doesn't match: %x %x", buf, big[:30]) + } +} + func BenchmarkWriteSlice1000Int32s(b *testing.B) { slice := make([]int32, 1000) buf := new(bytes.Buffer) @@ -722,6 +884,17 @@ func BenchmarkWriteSlice1000Int32s(b *testing.B) { b.StopTimer() } +func BenchmarkAppendSlice1000Int32s(b *testing.B) { + slice := make([]int32, 1000) + buf := make([]byte, 0, Size(slice)) + b.SetBytes(int64(cap(buf))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Append(buf, BigEndian, slice) + } + b.StopTimer() +} + func BenchmarkPutUint16(b *testing.B) { b.SetBytes(2) for i := 0; i < b.N; i++ {