mirror of
https://github.com/golang/go
synced 2024-09-05 16:04:50 +00:00
compress/flate: optimize huffman bit encoder
Part 1 of optimizing the deflater. This optimizes the bitwriter by: * Removing allocations. * Storing compound values for bit codes instead of 2 separate tables. * Accumulate 48 bits between writes instead of 24. * Inline bit flushing. This also contains code that will be used in later CL's (writeBlockDynamic, writeBlockHuff). Tests for Huffman bit writer encoding regressions has been added. name old speed new speed delta EncodeDigitsSpeed1e4-4 19.3MB/s ± 1% 21.6MB/s ± 1% +11.77% EncodeDigitsSpeed1e5-4 25.0MB/s ± 6% 30.7MB/s ± 1% +22.70% EncodeDigitsSpeed1e6-4 28.2MB/s ± 1% 32.3MB/s ± 1% +14.64% EncodeDigitsDefault1e4-4 13.3MB/s ± 0% 14.2MB/s ± 1% +7.07% EncodeDigitsDefault1e5-4 6.43MB/s ± 1% 6.64MB/s ± 1% +3.27% EncodeDigitsDefault1e6-4 5.81MB/s ± 0% 5.85MB/s ± 1% +0.69% EncodeDigitsCompress1e4-4 13.2MB/s ± 0% 14.4MB/s ± 0% +9.10% EncodeDigitsCompress1e5-4 6.40MB/s ± 1% 6.61MB/s ± 0% +3.20% EncodeDigitsCompress1e6-4 5.80MB/s ± 1% 5.90MB/s ± 1% +1.64% EncodeTwainSpeed1e4-4 18.4MB/s ± 1% 20.7MB/s ± 1% +12.72% EncodeTwainSpeed1e5-4 27.7MB/s ± 1% 31.0MB/s ± 1% +11.78% EncodeTwainSpeed1e6-4 29.1MB/s ± 0% 32.9MB/s ± 2% +13.25% EncodeTwainDefault1e4-4 12.4MB/s ± 0% 13.1MB/s ± 1% +5.88% EncodeTwainDefault1e5-4 7.52MB/s ± 1% 7.83MB/s ± 0% +4.19% EncodeTwainDefault1e6-4 7.08MB/s ± 1% 7.26MB/s ± 0% +2.54% EncodeTwainCompress1e4-4 12.0MB/s ± 1% 12.8MB/s ± 1% +6.70% EncodeTwainCompress1e5-4 5.96MB/s ± 1% 6.16MB/s ± 0% +3.27% EncodeTwainCompress1e6-4 5.37MB/s ± 0% 5.39MB/s ± 1% +0.47% >Allocations: benchmark old allocs new allocs delta BenchmarkEncodeDigitsSpeed1e4-4 50 0 -100.00% BenchmarkEncodeDigitsSpeed1e5-4 110 0 -100.00% BenchmarkEncodeDigitsSpeed1e6-4 1032 0 -100.00% BenchmarkEncodeDigitsDefault1e4-4 56 0 -100.00% BenchmarkEncodeDigitsDefault1e5-4 120 0 -100.00% BenchmarkEncodeDigitsDefault1e6-4 966 0 -100.00% BenchmarkEncodeDigitsCompress1e4-4 56 0 -100.00% BenchmarkEncodeDigitsCompress1e5-4 120 0 -100.00% BenchmarkEncodeDigitsCompress1e6-4 966 0 -100.00% BenchmarkEncodeTwainSpeed1e4-4 58 0 -100.00% BenchmarkEncodeTwainSpeed1e5-4 132 0 -100.00% BenchmarkEncodeTwainSpeed1e6-4 1082 0 -100.00% BenchmarkEncodeTwainDefault1e4-4 52 0 -100.00% BenchmarkEncodeTwainDefault1e5-4 126 0 -100.00% BenchmarkEncodeTwainDefault1e6-4 886 0 -100.00% BenchmarkEncodeTwainCompress1e4-4 52 0 -100.00% BenchmarkEncodeTwainCompress1e5-4 120 0 -100.00% BenchmarkEncodeTwainCompress1e6-4 880 0 -100.00% benchmark old bytes new bytes delta BenchmarkEncodeDigitsSpeed1e4-4 4288 2 -99.95% BenchmarkEncodeDigitsSpeed1e5-4 8896 15 -99.83% BenchmarkEncodeDigitsSpeed1e6-4 84098 153 -99.82% BenchmarkEncodeDigitsDefault1e4-4 4480 3 -99.93% BenchmarkEncodeDigitsDefault1e5-4 9216 76 -99.18% BenchmarkEncodeDigitsDefault1e6-4 73920 768 -98.96% BenchmarkEncodeDigitsCompress1e4-4 4480 3 -99.93% BenchmarkEncodeDigitsCompress1e5-4 9216 76 -99.18% BenchmarkEncodeDigitsCompress1e6-4 73920 768 -98.96% BenchmarkEncodeTwainSpeed1e4-4 4544 2 -99.96% BenchmarkEncodeTwainSpeed1e5-4 9600 15 -99.84% BenchmarkEncodeTwainSpeed1e6-4 77633 153 -99.80% BenchmarkEncodeTwainDefault1e4-4 4352 3 -99.93% BenchmarkEncodeTwainDefault1e5-4 9408 76 -99.19% BenchmarkEncodeTwainDefault1e6-4 65984 768 -98.84% BenchmarkEncodeTwainCompress1e4-4 4352 3 -99.93% BenchmarkEncodeTwainCompress1e5-4 9216 76 -99.18% BenchmarkEncodeTwainCompress1e6-4 65792 768 -98.83% Updates #14258 Change-Id: Ibaa97b9619743ad623094727228eb2ada1ec7f1f Reviewed-on: https://go-review.googlesource.com/19336 Reviewed-by: Nigel Tao <nigeltao@golang.org> Reviewed-by: Joe Tsai <joetsai@digital-static.net> Run-TryBot: Joe Tsai <joetsai@digital-static.net> TryBot-Result: Gobot Gobot <gobot@golang.org>
This commit is contained in:
parent
afdb8cff3e
commit
53984e5be2
|
@ -56,6 +56,8 @@ go src=..
|
|||
testdata
|
||||
+
|
||||
flate
|
||||
testdata
|
||||
+
|
||||
gzip
|
||||
testdata
|
||||
+
|
||||
|
|
|
@ -22,6 +22,17 @@ const (
|
|||
// The number of codegen codes.
|
||||
codegenCodeCount = 19
|
||||
badCode = 255
|
||||
|
||||
// bufferFlushSize indicates the buffer size
|
||||
// after which bytes are flushed to the writer.
|
||||
// Should preferably be a multiple of 6, since
|
||||
// we accumulate 6 bytes between writes to the buffer.
|
||||
bufferFlushSize = 240
|
||||
|
||||
// bufferSize is the actual output byte buffer size.
|
||||
// It must have additional headroom for a flush
|
||||
// which can contain up to 8 bytes.
|
||||
bufferSize = bufferFlushSize + 8
|
||||
)
|
||||
|
||||
// The number of extra bits needed by length code X - LENGTH_CODES_START.
|
||||
|
@ -70,9 +81,9 @@ type huffmanBitWriter struct {
|
|||
w io.Writer
|
||||
// Data waiting to be written is bytes[0:nbytes]
|
||||
// and then the low nbits of bits.
|
||||
bits uint32
|
||||
nbits uint32
|
||||
bytes [64]byte
|
||||
bits uint64
|
||||
nbits uint
|
||||
bytes [bufferSize]byte
|
||||
nbytes int
|
||||
literalFreq []int32
|
||||
offsetFreq []int32
|
||||
|
@ -92,52 +103,15 @@ func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
|
|||
codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
|
||||
codegenFreq: make([]int32, codegenCodeCount),
|
||||
literalEncoding: newHuffmanEncoder(maxNumLit),
|
||||
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
|
||||
codegenEncoding: newHuffmanEncoder(codegenCodeCount),
|
||||
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) reset(writer io.Writer) {
|
||||
w.w = writer
|
||||
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
|
||||
w.bytes = [64]byte{}
|
||||
for i := range w.codegen {
|
||||
w.codegen[i] = 0
|
||||
}
|
||||
for _, s := range [...][]int32{w.literalFreq, w.offsetFreq, w.codegenFreq} {
|
||||
for i := range s {
|
||||
s[i] = 0
|
||||
}
|
||||
}
|
||||
for _, enc := range [...]*huffmanEncoder{
|
||||
w.literalEncoding,
|
||||
w.offsetEncoding,
|
||||
w.codegenEncoding} {
|
||||
for i := range enc.code {
|
||||
enc.code[i] = 0
|
||||
}
|
||||
for i := range enc.codeBits {
|
||||
enc.codeBits[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) flushBits() {
|
||||
if w.err != nil {
|
||||
w.nbits = 0
|
||||
return
|
||||
}
|
||||
bits := w.bits
|
||||
w.bits >>= 16
|
||||
w.nbits -= 16
|
||||
n := w.nbytes
|
||||
w.bytes[n] = byte(bits)
|
||||
w.bytes[n+1] = byte(bits >> 8)
|
||||
if n += 2; n >= len(w.bytes) {
|
||||
_, w.err = w.w.Write(w.bytes[0:])
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
w.bytes = [bufferSize]byte{}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) flush() {
|
||||
|
@ -146,26 +120,41 @@ func (w *huffmanBitWriter) flush() {
|
|||
return
|
||||
}
|
||||
n := w.nbytes
|
||||
if w.nbits > 8 {
|
||||
for w.nbits != 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.bits >>= 8
|
||||
w.nbits -= 8
|
||||
n++
|
||||
}
|
||||
if w.nbits > 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.nbits = 0
|
||||
if w.nbits > 8 { // Avoid underflow
|
||||
w.nbits -= 8
|
||||
} else {
|
||||
w.nbits = 0
|
||||
}
|
||||
n++
|
||||
}
|
||||
w.bits = 0
|
||||
_, w.err = w.w.Write(w.bytes[0:n])
|
||||
_, w.err = w.w.Write(w.bytes[:n])
|
||||
w.nbytes = 0
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeBits(b, nb int32) {
|
||||
w.bits |= uint32(b) << w.nbits
|
||||
if w.nbits += uint32(nb); w.nbits >= 16 {
|
||||
w.flushBits()
|
||||
func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
|
||||
w.bits |= uint64(b) << w.nbits
|
||||
w.nbits += nb
|
||||
if w.nbits >= 48 {
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
n := w.nbytes
|
||||
w.bytes[n+0] = byte(bits)
|
||||
w.bytes[n+1] = byte(bits >> 8)
|
||||
w.bytes[n+2] = byte(bits >> 16)
|
||||
w.bytes[n+3] = byte(bits >> 24)
|
||||
w.bytes[n+4] = byte(bits >> 32)
|
||||
w.bytes[n+5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n >= bufferFlushSize {
|
||||
_, w.err = w.w.Write(w.bytes[:n])
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -174,17 +163,18 @@ func (w *huffmanBitWriter) writeBytes(bytes []byte) {
|
|||
return
|
||||
}
|
||||
n := w.nbytes
|
||||
if w.nbits == 8 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.nbits = 0
|
||||
n++
|
||||
}
|
||||
if w.nbits != 0 {
|
||||
if w.nbits&7 != 0 {
|
||||
w.err = InternalError("writeBytes with unfinished bits")
|
||||
return
|
||||
}
|
||||
for w.nbits != 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.bits >>= 8
|
||||
w.nbits -= 8
|
||||
n++
|
||||
}
|
||||
if n != 0 {
|
||||
_, w.err = w.w.Write(w.bytes[0:n])
|
||||
_, w.err = w.w.Write(w.bytes[:n])
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
@ -204,7 +194,8 @@ func (w *huffmanBitWriter) writeBytes(bytes []byte) {
|
|||
//
|
||||
// numLiterals The number of literals in literalEncoding
|
||||
// numOffsets The number of offsets in offsetEncoding
|
||||
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) {
|
||||
// litenc, offenc The literal and offset encoder to use
|
||||
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
|
||||
for i := range w.codegenFreq {
|
||||
w.codegenFreq[i] = 0
|
||||
}
|
||||
|
@ -214,8 +205,15 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) {
|
|||
// so far.
|
||||
codegen := w.codegen // cache
|
||||
// Copy the concatenated code sizes to codegen. Put a marker at the end.
|
||||
copy(codegen[0:numLiterals], w.literalEncoding.codeBits)
|
||||
copy(codegen[numLiterals:numLiterals+numOffsets], w.offsetEncoding.codeBits)
|
||||
cgnl := codegen[:numLiterals]
|
||||
for i := range cgnl {
|
||||
cgnl[i] = uint8(litEnc.codes[i].len)
|
||||
}
|
||||
|
||||
cgnl = codegen[numLiterals : numLiterals+numOffsets]
|
||||
for i := range cgnl {
|
||||
cgnl[i] = uint8(offEnc.codes[i].len)
|
||||
}
|
||||
codegen[numLiterals+numOffsets] = badCode
|
||||
|
||||
size := codegen[0]
|
||||
|
@ -284,11 +282,30 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int) {
|
|||
codegen[outIndex] = badCode
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeCode(code *huffmanEncoder, literal uint32) {
|
||||
func (w *huffmanBitWriter) writeCode(c hcode) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
w.writeBits(int32(code.code[literal]), int32(code.codeBits[literal]))
|
||||
w.bits |= uint64(c.code) << w.nbits
|
||||
w.nbits += uint(c.len)
|
||||
if w.nbits >= 48 {
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
n := w.nbytes
|
||||
w.bytes[n+0] = byte(bits)
|
||||
w.bytes[n+1] = byte(bits >> 8)
|
||||
w.bytes[n+2] = byte(bits >> 16)
|
||||
w.bytes[n+3] = byte(bits >> 24)
|
||||
w.bytes[n+4] = byte(bits >> 32)
|
||||
w.bytes[n+5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n >= bufferFlushSize {
|
||||
_, w.err = w.w.Write(w.bytes[:n])
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
}
|
||||
}
|
||||
|
||||
// Write the header of a dynamic Huffman block to the output stream.
|
||||
|
@ -310,7 +327,7 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n
|
|||
w.writeBits(int32(numCodegens-4), 4)
|
||||
|
||||
for i := 0; i < numCodegens; i++ {
|
||||
value := w.codegenEncoding.codeBits[codegenOrder[i]]
|
||||
value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
|
||||
w.writeBits(int32(value), 3)
|
||||
}
|
||||
|
||||
|
@ -321,8 +338,7 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n
|
|||
if codeWord == badCode {
|
||||
break
|
||||
}
|
||||
// The low byte contains the actual code to generate.
|
||||
w.writeCode(w.codegenEncoding, uint32(codeWord))
|
||||
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
|
||||
|
||||
switch codeWord {
|
||||
case 16:
|
||||
|
@ -371,48 +387,9 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
|
|||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
for i := range w.literalFreq {
|
||||
w.literalFreq[i] = 0
|
||||
}
|
||||
for i := range w.offsetFreq {
|
||||
w.offsetFreq[i] = 0
|
||||
}
|
||||
|
||||
n := len(tokens)
|
||||
tokens = tokens[0 : n+1]
|
||||
tokens[n] = endBlockMarker
|
||||
|
||||
for _, t := range tokens {
|
||||
switch t.typ() {
|
||||
case literalType:
|
||||
w.literalFreq[t.literal()]++
|
||||
case matchType:
|
||||
length := t.length()
|
||||
offset := t.offset()
|
||||
w.literalFreq[lengthCodesStart+lengthCode(length)]++
|
||||
w.offsetFreq[offsetCode(offset)]++
|
||||
}
|
||||
}
|
||||
|
||||
// get the number of literals
|
||||
numLiterals := len(w.literalFreq)
|
||||
for w.literalFreq[numLiterals-1] == 0 {
|
||||
numLiterals--
|
||||
}
|
||||
// get the number of offsets
|
||||
numOffsets := len(w.offsetFreq)
|
||||
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
|
||||
numOffsets--
|
||||
}
|
||||
if numOffsets == 0 {
|
||||
// We haven't found a single match. If we want to go with the dynamic encoding,
|
||||
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
|
||||
w.offsetFreq[0] = 1
|
||||
numOffsets = 1
|
||||
}
|
||||
|
||||
w.literalEncoding.generate(w.literalFreq, 15)
|
||||
w.offsetEncoding.generate(w.offsetFreq, 15)
|
||||
tokens = append(tokens, endBlockMarker)
|
||||
numLiterals, numOffsets := w.indexTokens(tokens)
|
||||
|
||||
storedBytes := 0
|
||||
if input != nil {
|
||||
|
@ -450,7 +427,7 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
|
|||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets)
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
|
||||
w.codegenEncoding.generate(w.codegenFreq, 7)
|
||||
numCodegens = len(w.codegenFreq)
|
||||
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
|
||||
|
@ -475,7 +452,7 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
|
|||
// Stored bytes?
|
||||
if storedSize < size {
|
||||
w.writeStoredHeader(storedBytes, eof)
|
||||
w.writeBytes(input[0:storedBytes])
|
||||
w.writeBytes(input[:storedBytes])
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -485,33 +462,222 @@ func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
|
|||
} else {
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
}
|
||||
|
||||
// Write the tokens.
|
||||
w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
|
||||
|
||||
}
|
||||
|
||||
// writeBlockDynamic encodes a block using a dynamic Huffman table.
|
||||
// This should be used if the symbols used have a disproportionate
|
||||
// histogram distribution.
|
||||
func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tokens = append(tokens, endBlockMarker)
|
||||
numLiterals, numOffsets := w.indexTokens(tokens)
|
||||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
|
||||
w.codegenEncoding.generate(w.codegenFreq, 7)
|
||||
numCodegens := len(w.codegenFreq)
|
||||
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
|
||||
numCodegens--
|
||||
}
|
||||
|
||||
// Write Huffman table.
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
|
||||
// Write the tokens.
|
||||
w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
|
||||
}
|
||||
|
||||
// indexTokens indexes a slice of tokens, and updates
|
||||
// literalFreq and offsetFreq, and generates literalEncoding
|
||||
// and offsetEncoding.
|
||||
// The number of literal and offset tokens is returned.
|
||||
func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
|
||||
for i := range w.literalFreq {
|
||||
w.literalFreq[i] = 0
|
||||
}
|
||||
for i := range w.offsetFreq {
|
||||
w.offsetFreq[i] = 0
|
||||
}
|
||||
|
||||
for _, t := range tokens {
|
||||
switch t.typ() {
|
||||
case literalType:
|
||||
w.writeCode(literalEncoding, t.literal())
|
||||
break
|
||||
case matchType:
|
||||
// Write the length
|
||||
length := t.length()
|
||||
lengthCode := lengthCode(length)
|
||||
w.writeCode(literalEncoding, lengthCode+lengthCodesStart)
|
||||
extraLengthBits := int32(lengthExtraBits[lengthCode])
|
||||
if extraLengthBits > 0 {
|
||||
extraLength := int32(length - lengthBase[lengthCode])
|
||||
w.writeBits(extraLength, extraLengthBits)
|
||||
}
|
||||
// Write the offset
|
||||
offset := t.offset()
|
||||
offsetCode := offsetCode(offset)
|
||||
w.writeCode(offsetEncoding, offsetCode)
|
||||
extraOffsetBits := int32(offsetExtraBits[offsetCode])
|
||||
if extraOffsetBits > 0 {
|
||||
extraOffset := int32(offset - offsetBase[offsetCode])
|
||||
w.writeBits(extraOffset, extraOffsetBits)
|
||||
}
|
||||
break
|
||||
default:
|
||||
panic("unknown token type: " + string(t))
|
||||
if t < matchType {
|
||||
w.literalFreq[t.literal()]++
|
||||
continue
|
||||
}
|
||||
length := t.length()
|
||||
offset := t.offset()
|
||||
w.literalFreq[lengthCodesStart+lengthCode(length)]++
|
||||
w.offsetFreq[offsetCode(offset)]++
|
||||
}
|
||||
|
||||
// get the number of literals
|
||||
numLiterals = len(w.literalFreq)
|
||||
for w.literalFreq[numLiterals-1] == 0 {
|
||||
numLiterals--
|
||||
}
|
||||
// get the number of offsets
|
||||
numOffsets = len(w.offsetFreq)
|
||||
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
|
||||
numOffsets--
|
||||
}
|
||||
if numOffsets == 0 {
|
||||
// We haven't found a single match. If we want to go with the dynamic encoding,
|
||||
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
|
||||
w.offsetFreq[0] = 1
|
||||
numOffsets = 1
|
||||
}
|
||||
w.literalEncoding.generate(w.literalFreq, 15)
|
||||
w.offsetEncoding.generate(w.offsetFreq, 15)
|
||||
return
|
||||
}
|
||||
|
||||
// writeTokens writes a slice of tokens to the output.
|
||||
// codes for literal and offset encoding must be supplied.
|
||||
func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
|
||||
for _, t := range tokens {
|
||||
if t < matchType {
|
||||
w.writeCode(leCodes[t.literal()])
|
||||
continue
|
||||
}
|
||||
// Write the length
|
||||
length := t.length()
|
||||
lengthCode := lengthCode(length)
|
||||
w.writeCode(leCodes[lengthCode+lengthCodesStart])
|
||||
extraLengthBits := uint(lengthExtraBits[lengthCode])
|
||||
if extraLengthBits > 0 {
|
||||
extraLength := int32(length - lengthBase[lengthCode])
|
||||
w.writeBits(extraLength, extraLengthBits)
|
||||
}
|
||||
// Write the offset
|
||||
offset := t.offset()
|
||||
offsetCode := offsetCode(offset)
|
||||
w.writeCode(oeCodes[offsetCode])
|
||||
extraOffsetBits := uint(offsetExtraBits[offsetCode])
|
||||
if extraOffsetBits > 0 {
|
||||
extraOffset := int32(offset - offsetBase[offsetCode])
|
||||
w.writeBits(extraOffset, extraOffsetBits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// huffOffset is a static offset encoder used for huffman only encoding.
|
||||
// It can be reused since we will not be encoding offset values.
|
||||
var huffOffset *huffmanEncoder
|
||||
|
||||
func init() {
|
||||
w := newHuffmanBitWriter(nil)
|
||||
w.offsetFreq[0] = 1
|
||||
huffOffset = newHuffmanEncoder(offsetCodeCount)
|
||||
huffOffset.generate(w.offsetFreq, 15)
|
||||
}
|
||||
|
||||
// writeBlockHuff encodes a block of bytes as either
|
||||
// Huffman encoded literals or uncompressed bytes if the
|
||||
// results only gains very little from compression.
|
||||
func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Clear histogram
|
||||
for i := range w.literalFreq {
|
||||
w.literalFreq[i] = 0
|
||||
}
|
||||
|
||||
// Add everything as literals
|
||||
histogram(input, w.literalFreq)
|
||||
|
||||
w.literalFreq[endBlockMarker] = 1
|
||||
|
||||
const numLiterals = endBlockMarker + 1
|
||||
const numOffsets = 1
|
||||
|
||||
w.literalEncoding.generate(w.literalFreq, 15)
|
||||
|
||||
// Figure out smallest code.
|
||||
// Always use dynamic Huffman or Store
|
||||
var numCodegens int
|
||||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
|
||||
w.codegenEncoding.generate(w.codegenFreq, 7)
|
||||
numCodegens = len(w.codegenFreq)
|
||||
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
|
||||
numCodegens--
|
||||
}
|
||||
headerSize := int64(3+5+5+4+(3*numCodegens)) +
|
||||
w.codegenEncoding.bitLength(w.codegenFreq) +
|
||||
int64(w.codegenFreq[16]*2) +
|
||||
int64(w.codegenFreq[17]*3) +
|
||||
int64(w.codegenFreq[18]*7)
|
||||
|
||||
// Includes EOB marker
|
||||
size := headerSize + w.literalEncoding.bitLength(w.literalFreq)
|
||||
|
||||
// Calculate stored size
|
||||
var storedSize int64 = math.MaxInt64
|
||||
var storedBytes = len(input)
|
||||
if storedBytes <= maxStoreBlockSize {
|
||||
storedSize = int64(storedBytes+5) * 8
|
||||
}
|
||||
|
||||
// Store bytes, if we don't get a reasonable improvement.
|
||||
if storedSize < (size + size>>4) {
|
||||
w.writeStoredHeader(storedBytes, eof)
|
||||
w.writeBytes(input)
|
||||
return
|
||||
}
|
||||
|
||||
// Huffman.
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
encoding := w.literalEncoding.codes
|
||||
n := w.nbytes
|
||||
for _, t := range input {
|
||||
// Bitwriting inlined, ~30% speedup
|
||||
c := encoding[t]
|
||||
w.bits |= uint64(c.code) << w.nbits
|
||||
w.nbits += uint(c.len)
|
||||
if w.nbits < 48 {
|
||||
continue
|
||||
}
|
||||
// Store 6 bytes
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
w.bytes[n+0] = byte(bits)
|
||||
w.bytes[n+1] = byte(bits >> 8)
|
||||
w.bytes[n+2] = byte(bits >> 16)
|
||||
w.bytes[n+3] = byte(bits >> 24)
|
||||
w.bytes[n+4] = byte(bits >> 32)
|
||||
w.bytes[n+5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n < bufferFlushSize {
|
||||
continue
|
||||
}
|
||||
_, w.err = w.w.Write(w.bytes[:n])
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
w.writeCode(encoding[endBlockMarker])
|
||||
}
|
||||
|
||||
// histogram accumulates a histogram of b in h.
|
||||
//
|
||||
// len(h) must be >= 256, and h's elements must be all zeroes.
|
||||
func histogram(b []byte, h []int32) {
|
||||
for _, t := range b {
|
||||
h[t]++
|
||||
}
|
||||
}
|
||||
|
|
366
src/compress/flate/huffman_bit_writer_test.go
Normal file
366
src/compress/flate/huffman_bit_writer_test.go
Normal file
File diff suppressed because one or more lines are too long
|
@ -9,9 +9,17 @@ import (
|
|||
"sort"
|
||||
)
|
||||
|
||||
// hcode is a huffman code with a bit code and bit length.
|
||||
type hcode struct {
|
||||
code, len uint16
|
||||
}
|
||||
|
||||
type huffmanEncoder struct {
|
||||
codeBits []uint8
|
||||
code []uint16
|
||||
codes []hcode
|
||||
freqcache []literalNode
|
||||
bitCount [17]int32
|
||||
lns byLiteral // stored to avoid repeated allocation in generate
|
||||
lfs byFreq // stored to avoid repeated allocation in generate
|
||||
}
|
||||
|
||||
type literalNode struct {
|
||||
|
@ -39,21 +47,26 @@ type levelInfo struct {
|
|||
needed int32
|
||||
}
|
||||
|
||||
// set sets the code and length of an hcode.
|
||||
func (h *hcode) set(code uint16, length uint16) {
|
||||
h.len = length
|
||||
h.code = code
|
||||
}
|
||||
|
||||
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
|
||||
|
||||
func newHuffmanEncoder(size int) *huffmanEncoder {
|
||||
return &huffmanEncoder{make([]uint8, size), make([]uint16, size)}
|
||||
return &huffmanEncoder{codes: make([]hcode, size)}
|
||||
}
|
||||
|
||||
// Generates a HuffmanCode corresponding to the fixed literal table
|
||||
func generateFixedLiteralEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(maxNumLit)
|
||||
codeBits := h.codeBits
|
||||
code := h.code
|
||||
codes := h.codes
|
||||
var ch uint16
|
||||
for ch = 0; ch < maxNumLit; ch++ {
|
||||
var bits uint16
|
||||
var size uint8
|
||||
var size uint16
|
||||
switch {
|
||||
case ch < 144:
|
||||
// size 8, 000110000 .. 10111111
|
||||
|
@ -75,19 +88,16 @@ func generateFixedLiteralEncoding() *huffmanEncoder {
|
|||
bits = ch + 192 - 280
|
||||
size = 8
|
||||
}
|
||||
codeBits[ch] = size
|
||||
code[ch] = reverseBits(bits, size)
|
||||
codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func generateFixedOffsetEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(30)
|
||||
codeBits := h.codeBits
|
||||
code := h.code
|
||||
codes := h.codes
|
||||
for ch := uint16(0); ch < 30; ch++ {
|
||||
codeBits[ch] = 5
|
||||
code[ch] = reverseBits(ch, 5)
|
||||
codes[ch] = hcode{code: reverseBits(ch, 5), len: 5}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
@ -99,7 +109,7 @@ func (h *huffmanEncoder) bitLength(freq []int32) int64 {
|
|||
var total int64
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
total += int64(f) * int64(h.codeBits[i])
|
||||
total += int64(f) * int64(h.codes[i].len)
|
||||
}
|
||||
}
|
||||
return total
|
||||
|
@ -220,7 +230,7 @@ func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
|
|||
panic("leafCounts[maxBits][maxBits] != n")
|
||||
}
|
||||
|
||||
bitCount := make([]int32, maxBits+1)
|
||||
bitCount := h.bitCount[:maxBits+1]
|
||||
bits := 1
|
||||
counts := &leafCounts[maxBits]
|
||||
for level := maxBits; level > 0; level-- {
|
||||
|
@ -246,10 +256,10 @@ func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalN
|
|||
// code, code + 1, .... The code values are
|
||||
// assigned in literal order (not frequency order).
|
||||
chunk := list[len(list)-int(bits):]
|
||||
sortByLiteral(chunk)
|
||||
|
||||
h.lns.sort(chunk)
|
||||
for _, node := range chunk {
|
||||
h.codeBits[node.literal] = uint8(n)
|
||||
h.code[node.literal] = reverseBits(code, uint8(n))
|
||||
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
|
||||
code++
|
||||
}
|
||||
list = list[0 : len(list)-int(bits)]
|
||||
|
@ -261,7 +271,13 @@ func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalN
|
|||
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
|
||||
// maxBits The maximum number of bits to use for any literal.
|
||||
func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
|
||||
list := make([]literalNode, len(freq)+1)
|
||||
if h.freqcache == nil {
|
||||
// Allocate a reusable buffer with the longest possible frequency table.
|
||||
// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
|
||||
// The largest of these is maxNumLit, so we allocate for that case.
|
||||
h.freqcache = make([]literalNode, maxNumLit+1)
|
||||
}
|
||||
list := h.freqcache[:len(freq)+1]
|
||||
// Number of non-zero literals
|
||||
count := 0
|
||||
// Set list to be the set of all non-zero literals and their frequencies
|
||||
|
@ -270,23 +286,23 @@ func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
|
|||
list[count] = literalNode{uint16(i), f}
|
||||
count++
|
||||
} else {
|
||||
h.codeBits[i] = 0
|
||||
list[count] = literalNode{}
|
||||
h.codes[i].len = 0
|
||||
}
|
||||
}
|
||||
// If freq[] is shorter than codeBits[], fill rest of codeBits[] with zeros
|
||||
h.codeBits = h.codeBits[0:len(freq)]
|
||||
list = list[0:count]
|
||||
list[len(freq)] = literalNode{}
|
||||
|
||||
list = list[:count]
|
||||
if count <= 2 {
|
||||
// Handle the small cases here, because they are awkward for the general case code. With
|
||||
// two or fewer literals, everything has bit length 1.
|
||||
for i, node := range list {
|
||||
// "list" is in order of increasing literal value.
|
||||
h.codeBits[node.literal] = 1
|
||||
h.code[node.literal] = uint16(i)
|
||||
h.codes[node.literal].set(uint16(i), 1)
|
||||
}
|
||||
return
|
||||
}
|
||||
sortByFreq(list)
|
||||
h.lfs.sort(list)
|
||||
|
||||
// Get the number of literals for each bit count
|
||||
bitCount := h.bitCounts(list, maxBits)
|
||||
|
@ -294,30 +310,35 @@ func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
|
|||
h.assignEncodingAndSize(bitCount, list)
|
||||
}
|
||||
|
||||
type literalNodeSorter struct {
|
||||
a []literalNode
|
||||
less func(i, j int) bool
|
||||
}
|
||||
type byLiteral []literalNode
|
||||
|
||||
func (s literalNodeSorter) Len() int { return len(s.a) }
|
||||
|
||||
func (s literalNodeSorter) Less(i, j int) bool {
|
||||
return s.less(i, j)
|
||||
}
|
||||
|
||||
func (s literalNodeSorter) Swap(i, j int) { s.a[i], s.a[j] = s.a[j], s.a[i] }
|
||||
|
||||
func sortByFreq(a []literalNode) {
|
||||
s := &literalNodeSorter{a, func(i, j int) bool {
|
||||
if a[i].freq == a[j].freq {
|
||||
return a[i].literal < a[j].literal
|
||||
}
|
||||
return a[i].freq < a[j].freq
|
||||
}}
|
||||
func (s *byLiteral) sort(a []literalNode) {
|
||||
*s = byLiteral(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func sortByLiteral(a []literalNode) {
|
||||
s := &literalNodeSorter{a, func(i, j int) bool { return a[i].literal < a[j].literal }}
|
||||
func (s byLiteral) Len() int { return len(s) }
|
||||
|
||||
func (s byLiteral) Less(i, j int) bool {
|
||||
return s[i].literal < s[j].literal
|
||||
}
|
||||
|
||||
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
type byFreq []literalNode
|
||||
|
||||
func (s *byFreq) sort(a []literalNode) {
|
||||
*s = byFreq(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s byFreq) Len() int { return len(s) }
|
||||
|
||||
func (s byFreq) Less(i, j int) bool {
|
||||
if s[i].freq == s[j].freq {
|
||||
return s[i].literal < s[j].literal
|
||||
}
|
||||
return s[i].freq < s[j].freq
|
||||
}
|
||||
|
||||
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
|
BIN
src/compress/flate/testdata/huffman-null-max.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-null-max.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.golden
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-null-max.in
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.in
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-null-max.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-null-max.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-null-max.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-pi.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-pi.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-pi.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-pi.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-pi.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-pi.golden
vendored
Normal file
Binary file not shown.
1
src/compress/flate/testdata/huffman-pi.in
vendored
Normal file
1
src/compress/flate/testdata/huffman-pi.in
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
|
BIN
src/compress/flate/testdata/huffman-pi.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-pi.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-pi.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-pi.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.golden
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.in
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.in
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-limit.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-limit.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-limit.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-limit.golden
vendored
Normal file
Binary file not shown.
4
src/compress/flate/testdata/huffman-rand-limit.in
vendored
Normal file
4
src/compress/flate/testdata/huffman-rand-limit.in
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
ř‹–vH
|
||||
…”%€ŻÂţŤč ë†É·ĹŢę}‹ç>Úß˙lsŢĚçmŤIGH°čžň1YŢ4´[ĺŕ 0Â<30>[|]o#©
|
||||
Ľ-#ľŮíul™ßýpfćîٱžn<C5BE>YŐÔ€Y<E282AC>w‰C8ÉŻ02š F=gn×ržN!OĆŕÔ{ŤĄö›kÜ*“w(ý´bÚ ç«kQC9/ ’lu>ô5ýC.÷¤uÚę›
|
BIN
src/compress/flate/testdata/huffman-rand-limit.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-limit.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-max.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-max.golden
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-rand-max.in
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-rand-max.in
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-shifts.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-shifts.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-shifts.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-shifts.golden
vendored
Normal file
Binary file not shown.
2
src/compress/flate/testdata/huffman-shifts.in
vendored
Normal file
2
src/compress/flate/testdata/huffman-shifts.in
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
|
||||
232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
|
BIN
src/compress/flate/testdata/huffman-shifts.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-shifts.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-shifts.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-shifts.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-text-shift.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-text-shift.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-text-shift.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-text-shift.golden
vendored
Normal file
Binary file not shown.
14
src/compress/flate/testdata/huffman-text-shift.in
vendored
Normal file
14
src/compress/flate/testdata/huffman-text-shift.in
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
//Copyright2009ThGoAuthor.Allrightrrvd.
|
||||
//UofthiourccodigovrndbyBSD-tyl
|
||||
//licnthtcnbfoundinthLICENSEfil.
|
||||
|
||||
pckgmin
|
||||
|
||||
import"o"
|
||||
|
||||
funcmin(){
|
||||
vrb=mk([]byt,65535)
|
||||
f,_:=o.Crt("huffmn-null-mx.in")
|
||||
f.Writ(b)
|
||||
}
|
||||
ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
|
BIN
src/compress/flate/testdata/huffman-text-shift.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-text-shift.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput
vendored
Normal file
Binary file not shown.
1
src/compress/flate/testdata/huffman-text.dyn.expect
vendored
Normal file
1
src/compress/flate/testdata/huffman-text.dyn.expect
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ì<>ÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐËÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
|
1
src/compress/flate/testdata/huffman-text.dyn.expect-noinput
vendored
Normal file
1
src/compress/flate/testdata/huffman-text.dyn.expect-noinput
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ì<>ÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐËÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
|
3
src/compress/flate/testdata/huffman-text.golden
vendored
Normal file
3
src/compress/flate/testdata/huffman-text.golden
vendored
Normal file
|
@ -0,0 +1,3 @@
|
|||
юAKС0ПСx╬ц÷·ZьзЯ╬LPьaн!┌x≥БADрЖI√&#I▀EЭНЧ гp]╒Lф©МЖ╞FПp≤╡ 1у88┤h⌠╒$┴ЁТ5SсЮ- ┌F66!┘)v┌.Т⌡0└Y╒≈М┘ШСц&Ее SсюыN|dё2:Ея
|
||||
t≤|К▒█ЮЫИxz9÷═╜⌠ ┴И╙╨▀ё╡·┴и▌в3┼░
|
||||
&&=Ыё╡╬╛Пц╢ UD▀=Fu▒РЦЁ]╡╛qЁшЩъUL+╫фНЖ╘>FQYйбLZ▐йoЭДэfTъ╣УEе╢рУ{╢Yй╤bЗeЗ
|
13
src/compress/flate/testdata/huffman-text.in
vendored
Normal file
13
src/compress/flate/testdata/huffman-text.in
vendored
Normal file
|
@ -0,0 +1,13 @@
|
|||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
var b = make([]byte, 65535)
|
||||
f, _ := os.Create("huffman-null-max.in")
|
||||
f.Write(b)
|
||||
}
|
1
src/compress/flate/testdata/huffman-text.wb.expect
vendored
Normal file
1
src/compress/flate/testdata/huffman-text.wb.expect
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ì<>ÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐËÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
|
1
src/compress/flate/testdata/huffman-text.wb.expect-noinput
vendored
Normal file
1
src/compress/flate/testdata/huffman-text.wb.expect-noinput
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ì<>ÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐËÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
|
BIN
src/compress/flate/testdata/huffman-zero.dyn.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-zero.dyn.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-zero.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-zero.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-zero.golden
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-zero.golden
vendored
Normal file
Binary file not shown.
1
src/compress/flate/testdata/huffman-zero.in
vendored
Normal file
1
src/compress/flate/testdata/huffman-zero.in
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
BIN
src/compress/flate/testdata/huffman-zero.wb.expect
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-zero.wb.expect
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/huffman-zero.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/huffman-zero.wb.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/null-long-match.dyn.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/null-long-match.dyn.expect-noinput
vendored
Normal file
Binary file not shown.
BIN
src/compress/flate/testdata/null-long-match.wb.expect-noinput
vendored
Normal file
BIN
src/compress/flate/testdata/null-long-match.wb.expect-noinput
vendored
Normal file
Binary file not shown.
Loading…
Reference in a new issue